Merge pull request #3321 from alephdata/release/3.15.1

Release/3.15.1 into main

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.15.0
+current_version = 3.15.1
 tag_name = {new_version}
 commit = True
 tag = True

.github/workflows/docs.yml

@@ -30,7 +30,7 @@ jobs:
           docs-path: ./docs
 
   deploy:
-    if: ${{ github.ref == 'refs/heads/main' }}
+    if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
     needs: build
     runs-on: ubuntu-latest
 

.github/workflows/pr.yml

@@ -0,0 +1,20 @@
+name: Check formatting
+
+on:
+  pull_request:
+    paths-ignore:
+      - "ui/**"
+      - "docs/**"
+
+jobs:
+  check-formatting:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install development dependencies
+        run: make dev
+
+      - name: Check code formatting
+        run: make format-check

Dockerfile

@@ -3,7 +3,7 @@ ENV DEBIAN_FRONTEND noninteractive
 
 # build-essential 
 RUN apt-get -qq -y update \
-    && apt-get -qq -y install locales \
+    && apt-get -qq --no-install-recommends -y install locales \
     ca-certificates postgresql-client libpq-dev curl jq \
     python3-pip python3-icu python3-psycopg2 \
     python3-lxml python3-crypto \
@@ -27,7 +27,7 @@ RUN pip3 install --no-cache-dir -q -r /tmp/requirements.txt
 COPY . /aleph
 WORKDIR /aleph
 ENV PYTHONPATH /aleph
-RUN pip install -q -e /aleph
+RUN pip install --no-cache-dir -q -e /aleph
 
 ENV ALEPH_WORD_FREQUENCY_URI=https://public.data.occrp.org/develop/models/word-frequencies/word_frequencies-v0.4.1.zip
 ENV ALEPH_FTM_COMPARE_MODEL_URI=https://public.data.occrp.org/develop/models/xref/glm_bernoulli_2e_wf-v0.4.1.pkl

Makefile

@@ -13,12 +13,6 @@ services:
 shell: services
 	$(APPDOCKER) /bin/bash
 
-shell-ui: services
-	$(UIDOCKER) /bin/bash
-
-shell-db: services
-	$(COMPOSE) exec postgres psql -U aleph
-
 # To run a single test file:
 # make test file=aleph/tests/test_manage.py
 test:

README.rst

@@ -78,7 +78,7 @@ Major, minor, patch releases
 3. Update translations using `make translate` 
 4. If you get npm errors, go into the ui folder and run `npm install`
 5. commit translations to `main` and push to remote
-6. run `bump2version release`. Note that bump2version won't show changes when you make the change, but it will work (see `git log` to check)
+6. run `bump2version --verbose --sign-tags release`. Note that bump2version won't show changes when you make the change, but it will work (see `git log` to check)
 7. push the tags to the remote with `git push --tags`
 8. push version bump to remote with `git push`
 9. merge `main` back into `develop`. Slightly unrelated to the release process but this is a good time to do it so that the new version numbers appear in `develop` as well

SECURITY.md

@@ -1,17 +1,13 @@
 # Security Policy
 
-## Supported Versions
+## Supported versions
 
 Aleph is a fast-moving project, developed through grant funding. We thus cannot provide specific 
 long-term support releases. Instead, we advise all implementors of the software to keep their
 installations up to date as much as they can.
 
-At the time of writing, versions 1.x and 2.x, and > 3.6 are completely discontinued.
+Please refer to our [Support Policy](SUPPORT.md) for more information about supported Aleph versions.
 
-## Reporting a Vulnerability
+## Reporting a vulnerability
 
-Low-grade security issues can be reported via GitHub issues. If you believe you have found a 
-critical security vulnerability, please consider contacting the Organized Crime and Corruption
-Reporting Project, the core maintainer of Aleph, directly via our responsible disclosure process:
-
-https://www.occrp.org/en/responsible-disclosure
+In order to report a security vulnerability, please contact the Organized Crime and Corruption Reporting Project (OCCRP), the core maintainer of Aleph, directly via [OCCRP’s Responsible Disclosure Policy](https://www.occrp.org/en/responsible-disclosure).

SUPPORT.md

@@ -1,18 +1,22 @@
-# Aleph Support Policy
+# Support Policy
 
 _Technology is neither good nor bad; nor is it neutral._ (Kranzberg)
 
 The objective of the Aleph project is to provide powerful software to those who do investigative work in the public interest.
 
 **We develop this technology following the open source model, and will continue to release our code to the public. At the same time, we have chosen to limit the scope of the community to whom we will provide support and engagement.**
 
-The maintainers of this project limit support and responses both on GitHub and in the Slack channel to authorised groups and individuals. In order to receive authorisation, we require that you disclose the manner in which you use our technology. **We will decide if that use falls within the intended uses of Aleph.** Examples of intended uses could include:
+## Eligible use cases
 
-- Professional investigative journalists.
+The maintainers of this project limit support and responses both on GitHub and in the Slack channel to authorised groups and individuals. In order to receive authorisation, we require that you disclose the manner in which you use our technology. **We will decide if that use falls within the intended uses of Aleph.**
+
+Examples of intended uses could include:
+
+- Professional investigative journalists
 - Activists, advocates and academics working in the public interest, and whose work is subject to an editorial policy
-- International bodies that have an investigative function.
+- International bodies that have an investigative function
 
-Support includes installation support, requests for new features or issues specific to your local Aleph installation. General bugs and contributions to the Aleph source code that can contribute to reliability of the system will be considered.
+Support includes installation support, requests for new features or issues specific to your local Aleph installation. Bug fixes and contributions to the Aleph source code that can contribute to reliability of the system will be considered.
 
 Please submit a description of your use case along with your name, affiliation and email address using one of the following channels:
 
@@ -22,6 +26,8 @@ Please submit a description of your use case along with your name, affiliation a
 
 Make sure you describe your goals, rather than the set of techniques that define your work (e.g. “investigations into human rights abuses in country X”, not “OSINT”). OCCRP is a non-profit organization. We do not offer commercial support or consulting services.
 
+## Supported versions
+
 The Aleph team supports feature versions for 12 months after the first major iteration of that version was released. For example, we support Aleph 3.12.x for 12 months after Aleph 3.12.0 was released. The Aleph team supports upgrades, but only from supported feature versions of the product. Support means helping to ensure that you can get your Aleph instance up and running. As we're a small team we don't have the capacity to backport bugs to supported versions. In the case of critical secruity vulnrabilities we'll endeavour to ensure that all currently supported versions, but we recommend administrators upgrade to the latest version as soon as possible.
 
 For versions that are supported, if you are having problems, you can reach out to us in Slack or by raising an issue in Github.

aleph.env.tmpl

@@ -63,9 +63,11 @@ ALEPH_OAUTH_SECRET=
 # Or, if 'ALEPH_ARCHIVE_TYPE' configuration is 's3':
 # ARCHIVE_TYPE=s3
 # ARCHIVE_BUCKET=
+# AWS_REGION=
+# Leave these next two keys empty if you prefer IAM Role-based auth
+# (see https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#id1)
 # AWS_ACCESS_KEY_ID=
 # AWS_SECRET_ACCESS_KEY=
-# AWS_REGION=
 
 # To use an external ElasticSearch service:
 # ALEPH_ELASTICSEARCH_URI=

aleph/index/util.py

@@ -116,19 +116,6 @@ def none_query(query=None):
     return query
 
 
-def query_string_query(field, query):
-    """Default config for querying the entity text."""
-    return {
-        "query_string": {
-            "query": query,
-            "lenient": True,
-            "fields": ensure_list(field),
-            "default_operator": "AND",
-            "minimum_should_match": "66%",
-        }
-    }
-
-
 def field_filter_query(field, values):
     """Need to define work-around for full-text fields."""
     values = ensure_list(values)

aleph/logic/alerts.py

@@ -6,7 +6,7 @@
 from aleph.core import db, es
 from aleph.model import Alert, Events, Entity
 from aleph.index.indexes import entities_read_index
-from aleph.index.util import unpack_result, authz_query, query_string_query
+from aleph.index.util import unpack_result, authz_query
 from aleph.logic.notifications import publish
 
 log = logging.getLogger(__name__)
@@ -74,7 +74,17 @@ def alert_query(alert, authz):
         "_source": {"includes": ["collection_id"]},
         "query": {
             "bool": {
-                "should": [query_string_query("text", alert.query)],
+                "should": [
+                    {
+                        "query_string": {
+                            "query": alert.query,
+                            "lenient": True,
+                            "fields": ["text"],
+                            "default_operator": "AND",
+                            "minimum_should_match": "66%",
+                        }
+                    }
+                ],
                 "filter": filters,
                 "minimum_should_match": 1,
             }

aleph/search/query.py

@@ -10,7 +10,6 @@
     field_filter_query,
     DATE_FORMAT,
     range_filter_query,
-    query_string_query,
     filter_text,
 )
 from aleph.search.result import SearchQueryResult
@@ -47,7 +46,15 @@ def __init__(self, parser):
     def get_text_query(self):
         query = []
         if self.parser.text:
-            qs = query_string_query(self.TEXT_FIELDS, self.parser.text)
+            qs = {
+                "query_string": {
+                    "query": self.parser.text,
+                    "lenient": True,
+                    "fields": self.TEXT_FIELDS,
+                    "default_operator": "AND",
+                    "minimum_should_match": "66%",
+                }
+            }
             query.append(qs)
         if self.parser.prefix:
             query.append(
@@ -219,12 +226,18 @@ def get_sort(self):
     def get_highlight(self):
         if not self.parser.highlight:
             return {}
-        query = query_string_query(self.HIGHLIGHT_FIELD, self.parser.highlight_text)
         return {
             "encoder": "html",
             "fields": {
                 self.HIGHLIGHT_FIELD: {
-                    "highlight_query": query,
+                    "highlight_query": {
+                        "query_string": {
+                            "query": self.parser.highlight_text,
+                            "lenient": True,
+                            "default_operator": "AND",
+                            "minimum_should_match": "66%",
+                        }
+                    },
                     "require_field_match": False,
                     "number_of_fragments": self.parser.highlight_count,
                     "fragment_size": self.parser.highlight_length,

aleph/tests/test_entities_api.py

@@ -125,6 +125,47 @@ def test_view_bookmarked(self):
         res = self.client.get(url, headers=headers)
         assert res.json["bookmarked"], res.json
 
+    def test_view_sanitize_html(self):
+        data = {
+            "schema": "HyperText",
+            "properties": {
+                "bodyHtml": "<style>body { color: red; }</style><p>Hello World!</p><script>alert('Ooops')</script>",
+            },
+        }
+
+        entity = self.create_entity(data, self.col)
+        index_entity(entity)
+
+        _, headers = self.login(is_admin=True)
+        url = f"/api/2/entities/{entity.id}"
+        res = self.client.get(url, headers=headers)
+
+        actual = res.json["safeHtml"]
+        expected = ["<html><body><div><p>Hello World!</p></div></body></html>"]
+        assert actual == expected, actual
+
+    def test_view_sanitize_html_multi_value(self):
+        data = {
+            "schema": "Email",
+            "properties": {
+                "bodyHtml": ["This is part 1.", "This is part 2."],
+            },
+        }
+
+        entity = self.create_entity(data, self.col)
+        index_entity(entity)
+
+        _, headers = self.login(is_admin=True)
+        url = f"/api/2/entities/{entity.id}"
+        res = self.client.get(url, headers=headers)
+
+        actual = res.json["safeHtml"]
+        expected = [
+            "<html><body><p>This is part 1.</p></body></html>",
+            "<html><body><p>This is part 2.</p></body></html>",
+        ]
+        assert actual == expected, actual
+
     def test_update(self):
         _, headers = self.login(is_admin=True)
         url = "/api/2/entities/%s" % self.id

aleph/translations/ar/LC_MESSAGES/aleph.po

@@ -13,7 +13,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2023-06-23 12:45+0200\n"
+"POT-Creation-Date: 2023-07-13 10:25+0000\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
 "Last-Translator: Mohammed AlKawmani <komani@arij.net>, 2020\n"
 "Language-Team: Arabic (https://app.transifex.com/aleph/teams/76591/ar/)\n"
@@ -228,7 +228,7 @@ msgstr ""
 msgid "{{export}} is ready for download"
 msgstr ""
 
-#: aleph/search/__init__.py:52
+#: aleph/search/__init__.py:76
 msgid "No schema is specified for the query."
 msgstr "لم يتم تحديد مخطط للاستعلام."
 
@@ -342,23 +342,23 @@ msgstr "اسم مخطط غير صحيح : 1%s"
 msgid "Invalid date: %s"
 msgstr "تاريخ خاطئ 1%s"
 
-#: aleph/views/base_api.py:222
+#: aleph/views/base_api.py:226
 msgid "You are not authorized to do this."
 msgstr "غير مخول للقيام بذلك."
 
-#: aleph/views/base_api.py:231
+#: aleph/views/base_api.py:235
 msgid "This path does not exist."
 msgstr "هذا المسار غير موجود."
 
-#: aleph/views/base_api.py:238
+#: aleph/views/base_api.py:242
 msgid "Internal server error."
 msgstr "خطأ في الخادم الداخلي."
 
-#: aleph/views/base_api.py:252
+#: aleph/views/base_api.py:256
 msgid "Access token is invalid."
 msgstr "رمز الدخول غير صحيح"
 
-#: aleph/views/entities_api.py:172
+#: aleph/views/entities_api.py:173
 #, python-format
 msgid "Search: %s"
 msgstr ""

aleph/translations/bs/LC_MESSAGES/aleph.po

@@ -14,7 +14,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2023-06-23 12:45+0200\n"
+"POT-Creation-Date: 2023-07-13 10:25+0000\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
 "Last-Translator: Tajna Biscevic <tajna@occrp.org>, 2019\n"
 "Language-Team: Bosnian (https://app.transifex.com/aleph/teams/76591/bs/)\n"
@@ -229,7 +229,7 @@ msgstr ""
 msgid "{{export}} is ready for download"
 msgstr ""
 
-#: aleph/search/__init__.py:52
+#: aleph/search/__init__.py:76
 msgid "No schema is specified for the query."
 msgstr "Nije navedena shema za upit."
 
@@ -343,23 +343,23 @@ msgstr ""
 msgid "Invalid date: %s"
 msgstr ""
 
-#: aleph/views/base_api.py:222
+#: aleph/views/base_api.py:226
 msgid "You are not authorized to do this."
 msgstr "Niste ovlašteni da ovo uradite."
 
-#: aleph/views/base_api.py:231
+#: aleph/views/base_api.py:235
 msgid "This path does not exist."
 msgstr "Ova putanja ne postoji."
 
-#: aleph/views/base_api.py:238
+#: aleph/views/base_api.py:242
 msgid "Internal server error."
 msgstr "Interna greška servera."
 
-#: aleph/views/base_api.py:252
+#: aleph/views/base_api.py:256
 msgid "Access token is invalid."
 msgstr ""
 
-#: aleph/views/entities_api.py:172
+#: aleph/views/entities_api.py:173
 #, python-format
 msgid "Search: %s"
 msgstr ""