diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 7e33297569..75ec214271 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 3.15.0
+current_version = 3.15.1
tag_name = {new_version}
commit = True
tag = True
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 99bf702dc0..4ca9cc9fcd 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -30,7 +30,7 @@ jobs:
docs-path: ./docs
deploy:
- if: ${{ github.ref == 'refs/heads/main' }}
+ if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
needs: build
runs-on: ubuntu-latest
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
new file mode 100644
index 0000000000..2995d2fb2b
--- /dev/null
+++ b/.github/workflows/pr.yml
@@ -0,0 +1,20 @@
+name: Check formatting
+
+on:
+ pull_request:
+ paths-ignore:
+ - "ui/**"
+ - "docs/**"
+
+jobs:
+ check-formatting:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Install development dependencies
+ run: make dev
+
+ - name: Check code formatting
+ run: make format-check
diff --git a/Dockerfile b/Dockerfile
index 744e85b131..05ebeaa78f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,7 +3,7 @@ ENV DEBIAN_FRONTEND noninteractive
# build-essential
RUN apt-get -qq -y update \
- && apt-get -qq -y install locales \
+ && apt-get -qq --no-install-recommends -y install locales \
ca-certificates postgresql-client libpq-dev curl jq \
python3-pip python3-icu python3-psycopg2 \
python3-lxml python3-crypto \
@@ -27,7 +27,7 @@ RUN pip3 install --no-cache-dir -q -r /tmp/requirements.txt
COPY . /aleph
WORKDIR /aleph
ENV PYTHONPATH /aleph
-RUN pip install -q -e /aleph
+RUN pip install --no-cache-dir -q -e /aleph
ENV ALEPH_WORD_FREQUENCY_URI=https://public.data.occrp.org/develop/models/word-frequencies/word_frequencies-v0.4.1.zip
ENV ALEPH_FTM_COMPARE_MODEL_URI=https://public.data.occrp.org/develop/models/xref/glm_bernoulli_2e_wf-v0.4.1.pkl
diff --git a/Makefile b/Makefile
index 2b9bd710fa..a9927e9779 100644
--- a/Makefile
+++ b/Makefile
@@ -13,12 +13,6 @@ services:
shell: services
$(APPDOCKER) /bin/bash
-shell-ui: services
- $(UIDOCKER) /bin/bash
-
-shell-db: services
- $(COMPOSE) exec postgres psql -U aleph
-
# To run a single test file:
# make test file=aleph/tests/test_manage.py
test:
diff --git a/README.rst b/README.rst
index c90c2d2a29..90821e0dc1 100644
--- a/README.rst
+++ b/README.rst
@@ -78,7 +78,7 @@ Major, minor, patch releases
3. Update translations using `make translate`
4. If you get npm errors, go into the ui folder and run `npm install`
5. commit translations to `main` and push to remote
-6. run `bump2version release`. Note that bump2version won't show changes when you make the change, but it will work (see `git log` to check)
+6. run `bump2version --verbose --sign-tags release`. Note that bump2version won't show changes when you make the change, but it will work (see `git log` to check)
7. push the tags to the remote with `git push --tags`
8. push version bump to remote with `git push`
9. merge `main` back into `develop`. Slightly unrelated to the release process but this is a good time to do it so that the new version numbers appear in `develop` as well
diff --git a/SECURITY.md b/SECURITY.md
index ded5161a91..0342074d6e 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -1,17 +1,13 @@
# Security Policy
-## Supported Versions
+## Supported versions
Aleph is a fast-moving project, developed through grant funding. We thus cannot provide specific
long-term support releases. Instead, we advise all implementors of the software to keep their
installations up to date as much as they can.
-At the time of writing, versions 1.x and 2.x, and > 3.6 are completely discontinued.
+Please refer to our [Support Policy](SUPPORT.md) for more information about supported Aleph versions.
-## Reporting a Vulnerability
+## Reporting a vulnerability
-Low-grade security issues can be reported via GitHub issues. If you believe you have found a
-critical security vulnerability, please consider contacting the Organized Crime and Corruption
-Reporting Project, the core maintainer of Aleph, directly via our responsible disclosure process:
-
-https://www.occrp.org/en/responsible-disclosure
+In order to report a security vulnerability, please contact the Organized Crime and Corruption Reporting Project (OCCRP), the core maintainer of Aleph, directly via [OCCRP’s Responsible Disclosure Policy](https://www.occrp.org/en/responsible-disclosure).
diff --git a/SUPPORT.md b/SUPPORT.md
index 02f30bdc23..082907907f 100644
--- a/SUPPORT.md
+++ b/SUPPORT.md
@@ -1,4 +1,4 @@
-# Aleph Support Policy
+# Support Policy
_Technology is neither good nor bad; nor is it neutral._ (Kranzberg)
@@ -6,13 +6,17 @@ The objective of the Aleph project is to provide powerful software to those who
**We develop this technology following the open source model, and will continue to release our code to the public. At the same time, we have chosen to limit the scope of the community to whom we will provide support and engagement.**
-The maintainers of this project limit support and responses both on GitHub and in the Slack channel to authorised groups and individuals. In order to receive authorisation, we require that you disclose the manner in which you use our technology. **We will decide if that use falls within the intended uses of Aleph.** Examples of intended uses could include:
+## Eligible use cases
-- Professional investigative journalists.
+The maintainers of this project limit support and responses both on GitHub and in the Slack channel to authorised groups and individuals. In order to receive authorisation, we require that you disclose the manner in which you use our technology. **We will decide if that use falls within the intended uses of Aleph.**
+
+Examples of intended uses could include:
+
+- Professional investigative journalists
- Activists, advocates and academics working in the public interest, and whose work is subject to an editorial policy
-- International bodies that have an investigative function.
+- International bodies that have an investigative function
-Support includes installation support, requests for new features or issues specific to your local Aleph installation. General bugs and contributions to the Aleph source code that can contribute to reliability of the system will be considered.
+Support includes installation support, requests for new features or issues specific to your local Aleph installation. Bug fixes and contributions to the Aleph source code that can contribute to reliability of the system will be considered.
Please submit a description of your use case along with your name, affiliation and email address using one of the following channels:
@@ -22,6 +26,8 @@ Please submit a description of your use case along with your name, affiliation a
Make sure you describe your goals, rather than the set of techniques that define your work (e.g. “investigations into human rights abuses in country X”, not “OSINT”). OCCRP is a non-profit organization. We do not offer commercial support or consulting services.
+## Supported versions
+
The Aleph team supports feature versions for 12 months after the first major iteration of that version was released. For example, we support Aleph 3.12.x for 12 months after Aleph 3.12.0 was released. The Aleph team supports upgrades, but only from supported feature versions of the product. Support means helping to ensure that you can get your Aleph instance up and running. As we're a small team we don't have the capacity to backport bugs to supported versions. In the case of critical secruity vulnrabilities we'll endeavour to ensure that all currently supported versions, but we recommend administrators upgrade to the latest version as soon as possible.
For versions that are supported, if you are having problems, you can reach out to us in Slack or by raising an issue in Github.
diff --git a/aleph.env.tmpl b/aleph.env.tmpl
index fc32e650fe..da2f4589a1 100644
--- a/aleph.env.tmpl
+++ b/aleph.env.tmpl
@@ -63,9 +63,11 @@ ALEPH_OAUTH_SECRET=
# Or, if 'ALEPH_ARCHIVE_TYPE' configuration is 's3':
# ARCHIVE_TYPE=s3
# ARCHIVE_BUCKET=
+# AWS_REGION=
+# Leave these next two keys empty if you prefer IAM Role-based auth
+# (see https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#id1)
# AWS_ACCESS_KEY_ID=
# AWS_SECRET_ACCESS_KEY=
-# AWS_REGION=
# To use an external ElasticSearch service:
# ALEPH_ELASTICSEARCH_URI=
diff --git a/aleph/index/util.py b/aleph/index/util.py
index 39ac7b0676..f78c3da529 100644
--- a/aleph/index/util.py
+++ b/aleph/index/util.py
@@ -116,19 +116,6 @@ def none_query(query=None):
return query
-def query_string_query(field, query):
- """Default config for querying the entity text."""
- return {
- "query_string": {
- "query": query,
- "lenient": True,
- "fields": ensure_list(field),
- "default_operator": "AND",
- "minimum_should_match": "66%",
- }
- }
-
-
def field_filter_query(field, values):
"""Need to define work-around for full-text fields."""
values = ensure_list(values)
diff --git a/aleph/logic/alerts.py b/aleph/logic/alerts.py
index 7eae7d0bd0..adac32e646 100644
--- a/aleph/logic/alerts.py
+++ b/aleph/logic/alerts.py
@@ -6,7 +6,7 @@
from aleph.core import db, es
from aleph.model import Alert, Events, Entity
from aleph.index.indexes import entities_read_index
-from aleph.index.util import unpack_result, authz_query, query_string_query
+from aleph.index.util import unpack_result, authz_query
from aleph.logic.notifications import publish
log = logging.getLogger(__name__)
@@ -74,7 +74,17 @@ def alert_query(alert, authz):
"_source": {"includes": ["collection_id"]},
"query": {
"bool": {
- "should": [query_string_query("text", alert.query)],
+ "should": [
+ {
+ "query_string": {
+ "query": alert.query,
+ "lenient": True,
+ "fields": ["text"],
+ "default_operator": "AND",
+ "minimum_should_match": "66%",
+ }
+ }
+ ],
"filter": filters,
"minimum_should_match": 1,
}
diff --git a/aleph/search/query.py b/aleph/search/query.py
index 2175cde1c5..d393159384 100644
--- a/aleph/search/query.py
+++ b/aleph/search/query.py
@@ -10,7 +10,6 @@
field_filter_query,
DATE_FORMAT,
range_filter_query,
- query_string_query,
filter_text,
)
from aleph.search.result import SearchQueryResult
@@ -47,7 +46,15 @@ def __init__(self, parser):
def get_text_query(self):
query = []
if self.parser.text:
- qs = query_string_query(self.TEXT_FIELDS, self.parser.text)
+ qs = {
+ "query_string": {
+ "query": self.parser.text,
+ "lenient": True,
+ "fields": self.TEXT_FIELDS,
+ "default_operator": "AND",
+ "minimum_should_match": "66%",
+ }
+ }
query.append(qs)
if self.parser.prefix:
query.append(
@@ -219,12 +226,18 @@ def get_sort(self):
def get_highlight(self):
if not self.parser.highlight:
return {}
- query = query_string_query(self.HIGHLIGHT_FIELD, self.parser.highlight_text)
return {
"encoder": "html",
"fields": {
self.HIGHLIGHT_FIELD: {
- "highlight_query": query,
+ "highlight_query": {
+ "query_string": {
+ "query": self.parser.highlight_text,
+ "lenient": True,
+ "default_operator": "AND",
+ "minimum_should_match": "66%",
+ }
+ },
"require_field_match": False,
"number_of_fragments": self.parser.highlight_count,
"fragment_size": self.parser.highlight_length,
diff --git a/aleph/tests/test_entities_api.py b/aleph/tests/test_entities_api.py
index 86b87f5e9d..0ac46aa4ac 100644
--- a/aleph/tests/test_entities_api.py
+++ b/aleph/tests/test_entities_api.py
@@ -125,6 +125,47 @@ def test_view_bookmarked(self):
res = self.client.get(url, headers=headers)
assert res.json["bookmarked"], res.json
+ def test_view_sanitize_html(self):
+ data = {
+ "schema": "HyperText",
+ "properties": {
+ "bodyHtml": "
+ Aleph is a data platform created and maintained by the Organized Crime and
+ Corruption Reporting Project (OCCRP). The tool was built to help
+ investigative journalists track people and companies, usually as
+ part of their corruption investigations.
+
+ Are you in a hurry and don't have time to read? Watch our short video on
+ how to create an Aleph account.
+
+
+
+
diff --git a/docs/src/layouts/DocsLayout.astro b/docs/src/layouts/DefaultLayout.astro
similarity index 73%
rename from docs/src/layouts/DocsLayout.astro
rename to docs/src/layouts/DefaultLayout.astro
index d2a8abaea5..9195ec7f6c 100644
--- a/docs/src/layouts/DocsLayout.astro
+++ b/docs/src/layouts/DefaultLayout.astro
@@ -3,7 +3,10 @@ import { DocsLayout } from 'astro-theme-docs/components';
import options from '../options.json';
---
-
+
diff --git a/docs/src/layouts/DevelopersLayout.astro b/docs/src/layouts/DevelopersLayout.astro
new file mode 100644
index 0000000000..1142445991
--- /dev/null
+++ b/docs/src/layouts/DevelopersLayout.astro
@@ -0,0 +1,14 @@
+---
+import { DocsLayout } from 'astro-theme-docs/components';
+import options from '../options.json';
+---
+
+
+
+
+
+
+
diff --git a/docs/src/layouts/UsersLayout.astro b/docs/src/layouts/UsersLayout.astro
new file mode 100644
index 0000000000..a1a2f50315
--- /dev/null
+++ b/docs/src/layouts/UsersLayout.astro
@@ -0,0 +1,14 @@
+---
+import { DocsLayout } from 'astro-theme-docs/components';
+import options from '../options.json';
+---
+
+
+
+
+
+
+
diff --git a/docs/src/options.json b/docs/src/options.json
index 11b39e6c2d..c924822c47 100644
--- a/docs/src/options.json
+++ b/docs/src/options.json
@@ -9,67 +9,65 @@
"apiKey": "b462393bca678070ceb9abf533aeeb0b",
"indexName": "aleph-occrp"
},
- "nav": {
- "header": {
- "items": []
- },
- "sidebar": {
+ "translations": {
+ "defaultLanguageName": "English",
+ "languages": {
+ "es": "Español",
+ "pt": "Português"
+ }
+ },
+ "header": {
+ "items": [
+ { "title": "Users", "link": "/users" },
+ { "title": "Developers", "link": "/developers" }
+ ]
+ },
+ "sidebar": {
+ "users": {
"sections": [
{
- "title": "About",
+ "title": "Getting started",
"items": [
- { "slug": "/" },
- { "slug": "/how-aleph-is-used" },
- { "slug": "/get-in-touch" }
+ { "slug": "/users/getting-started/account" },
+ { "slug": "/users/getting-started/key-terms" },
+ { "slug": "/users/getting-started/homepage" }
]
},
{
- "title": "Users",
+ "title": "Search",
"items": [
- { "slug": "/guide/getting-started" },
- { "slug": "/guide/the-basics" },
- {
- "slug": "/guide/search",
- "children": [
- { "slug": "/guide/search/anatomy-of-a-search" },
- { "slug": "/guide/search/advanced-search-methods" },
- { "slug": "/guide/search/filtering-your-search-results" },
- { "slug": "/guide/search/searching-within-other-contexts" },
- { "slug": "/guide/search/searching-for-a-dataset" }
- ]
- },
- {
- "slug": "/guide/building-out-your-investigation",
- "children": [
- {
- "slug": "/guide/building-out-your-investigation/creating-an-investigation"
- },
- {
- "slug": "/guide/building-out-your-investigation/uploading-documents"
- },
- {
- "slug": "/guide/building-out-your-investigation/network-diagrams"
- },
- {
- "slug": "/guide/building-out-your-investigation/using-the-table-editor"
- },
- {
- "slug": "/guide/building-out-your-investigation/generating-multiple-entities-from-a-list"
- },
- {
- "slug": "/guide/building-out-your-investigation/cross-referencing"
- }
- ]
- },
- { "slug": "/guide/aleph-data-desktop" },
- { "slug": "/guide/reconciliation" },
- { "slug": "/guide/frequently-asked-questions" }
+ { "slug": "/users/search/basics" },
+ { "slug": "/users/search/advanced" },
+ { "slug": "/users/search/datasets" }
+ ]
+ },
+ {
+ "title": "Investigations",
+ "items": [
+ { "slug": "/users/investigations/overview" },
+ { "slug": "/users/investigations/create" },
+ { "slug": "/users/investigations/manage-access" },
+ { "slug": "/users/investigations/uploading-documents" },
+ { "slug": "/users/investigations/entity-editor" },
+ { "slug": "/users/investigations/cross-referencing" },
+ { "slug": "/users/investigations/network-diagrams" }
]
},
{
- "title": "Developers & Admins",
+ "title": "FAQ",
+ "items": [{ "slug": "/users/faq/account" }]
+ },
+ {
+ "title": "Advanced",
+ "items": [{ "slug": "/users/advanced/reconciliation" }]
+ }
+ ]
+ },
+ "developers": {
+ "sections": [
+ {
+ "title": "Tech Docs",
"items": [
- { "slug": "/developers/intro" },
{ "slug": "/developers/installation" },
{
"slug": "/developers/followthemoney",
diff --git a/docs/src/pages/about/index.mdx b/docs/src/pages/about/index.mdx
new file mode 100644
index 0000000000..aeb6a7b4ac
--- /dev/null
+++ b/docs/src/pages/about/index.mdx
@@ -0,0 +1,69 @@
+---
+layout: '@layouts/DefaultLayout.astro'
+title: About Aleph
+---
+
+# About Aleph
+
+Aleph is a data platform created and maintained by the Organized Crime and Corruption Reporting Project (OCCRP). The tool was built to help investigative journalists track people and companies, usually as part of their corruption investigations. In addition to being a tool for searching and finding insights in large volumes of data, Aleph provides an easy and secure way to upload documents and leaks, and create network diagrams, timelines and lists of people that you can compare against hundreds of datasets.
+
+## Why was Aleph created?
+
+* **To search structured data (e.g. a database) and unstructured data (e.g. a bunch of documents)**.
+* **To cross-reference** between different datasets ("Who are all the politicians in my country that are mentioned in this leak?").
+* **To manage user access**, but also flexible sharing within cross-border teams.
+* **To visually explore data** through network diagrams and timelines.
+* **To regularly update datasets** in an automated way using Aleph’s rich API that enables other tools, such as web crawlers, to integrate with it.
+
+## Who is using Aleph?
+
+Dozens of public interest organizations run Aleph instances in-house. A few have public instances such as:
+
+* Balkans Investigative Reporting Network (BIRN) – [Research Desk](https://source.bird.tools/)
+* DDoSecrets – [Hunter Memorial Library](https://hunter.ddosecrets.com/)
+* Code for Africa – [gazeti.africa](https://gazeti.africa/)
+
+Some of the organizations with in-house instances:
+
+* Global Witness
+* Süddeutsche Zeitung (SZ)
+* Swedish Radio (SR)
+* Norddeutscher Rundfunk (NDR)
+
+
+ If you have used Aleph for a project or you have your own Aleph instance, we’d love to hear from you! Feel free to contact us [using this form](https://form.asana.com/?k=8sddelbvZCbEb3h_pfUchQ&d=24418422500834) to share how you are using Aleph or to give feedback! View our [Aleph Support Policy](https://github.com/alephdata/aleph/blob/main/SUPPORT.md) for more information on how OCCRP can help.
+
+
+## Who has funded the development?
+
+We are very grateful to the various donors who have made the development of Aleph possible. Some of these funders include:
+
+* Early on the [Google Digital News Initiative](https://newsinitiative.withgoogle.com/) supported OCCRP with a [large-scale grant](https://newsinitiative.withgoogle.com/dnifund/dni-projects/turnkey-data-platform-investigative-teams/) to build Aleph. Google Ideas (now Jigsaw, an Alphabet unit) also provided funding for the [Investigative Dashboard](https://id.occrp.org/), some of which helped to kickstart this project.
+
+* The [National Endowment for Democracy](https://www.ned.org/) (NED) and the [NLnet Foundation](https://nlnet.nl/) provided Aleph specific funding.
+
+* Many of OCCRP's core grants have financed Aleph, for example from the [US Agency for International Development](https://www.usaid.gov/) (USAID), the [US Department of State](https://www.state.gov/bureaus-offices/under-secretary-for-civilian-security-democracy-and-human-rights/bureau-of-democracy-human-rights-and-labor/) (DRL), [Open Society Foundations](https://www.opensocietyfoundations.org/) (OSF), [Luminate](https://www.luminategroup.com/), Sigrid Rausing, SIDA, and Oak Foundation.
+
+You can find a full list of OCCRP’s supporters on [our website](https://www.occrp.org/en/aboutus/who-supports-our-work).
+
+Besides grants to OCCRP, the following organizations have contributed to the development of Aleph:
+
+* The initial prototyping of Aleph was supported by the [International Center for Journalists](https://www.icfj.org/) (ICFJ) through a [Knight International Journalism Fellowship](https://www.icfj.org/our-work/knight/profiles/friedrich-lindenberg) in 2014/2015.
+* EU Project [ODINE](https://opendataincubator.eu/) supported the use of Aleph by [OpenOil UG](https://opendataincubator.eu/category/openoil/).
+* [Data Science for Social Good](https://www.dssgfellowship.org/) (DSSG) worked with us to produce a machine learning algorithm to help with document categorization.
+
+
+ If you would like to support OCCRP's development of Aleph, please contact our Partnerships team via info@occrp.org.
+
+
+## How can I contribute?
+
+Aleph is an open source project, we're very excited for contributions from people or organizations who see benefit in helping to refine or extend our tools. Here are some of the ways you could help:
+
+* If you are a **native speaker** of a language that isn't supported by the Aleph user interface, join the [Transifex project](https://www.transifex.com/aleph/) and help us provide the software in that language.
+
+* If you're a **coder** (Python, JavaScript/React.js), please refer to our [contribution guide](https://github.com/alephdata/aleph/blob/main/CONTRIBUTING.md) to get started.
+
+* If you're an **open data enthusiast** or data scraper, you can convert data from a public source to our FollowTheMoney data format and share the result with the community. Consider contributing a memorious scraper to the Aleph data commons!
+
+* Any **graphic designers or visual artists** could help us immensely, in particular by improving the set of icons available inside the application.
diff --git a/docs/src/pages/developers/adding-text-processors.mdx b/docs/src/pages/developers/adding-text-processors.mdx
index b6071d6475..f6462878df 100644
--- a/docs/src/pages/developers/adding-text-processors.mdx
+++ b/docs/src/pages/developers/adding-text-processors.mdx
@@ -1,5 +1,5 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: Adding text processors
---
diff --git a/docs/src/pages/developers/alephclient.mdx b/docs/src/pages/developers/alephclient.mdx
index 7347be69bc..3cf76c0b09 100644
--- a/docs/src/pages/developers/alephclient.mdx
+++ b/docs/src/pages/developers/alephclient.mdx
@@ -1,5 +1,5 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: alephclient CLI
---
diff --git a/docs/src/pages/developers/changelog.md b/docs/src/pages/developers/changelog.md
deleted file mode 100644
index 5895e43118..0000000000
--- a/docs/src/pages/developers/changelog.md
+++ /dev/null
@@ -1,307 +0,0 @@
----
-description: >-
- This document is intended to make easier for operators of Aleph instances to
- follow the development and perform upgrades to their local installation.
----
-
-# Changelog
-
-## 3.13.0 (2022-11-21)
-
-- Library upgrades
-- Minor fixes
-- Updated Aleph to use Blueprint 4 (previous version was blueprint3)
-- react-ftm has now been re-integrated with Aleph. The react-ftm library will now be deprecated
-- Fixed the icon colour and alignment in network diagrams
-- Fixed issue with scroll position resetting
-- Fix investigation sidebar width
-- Handle off-by-one bug in table viewer
-
-## 3.12.7 (2022-10-11)
-
-- Library upgrades
-- Minor fixes
-- Updated to react-ftm v2.6.8, this release contains some small bugfixes and accessibility improvements
-- improvements to highlighting in search results. We now display highlighted results within the raw text documents.
-- Better handling of status messages without the need for Aleph re-deployment
-- Updated contact details
-- Updated custom settings in Aleph. There is now a mechanism for setting these via environment variables.
-
-## 3.12.6 (2022-07-25)
-
-- Develop QoL updates. Introduced prettier and eslint builds for github. Update code to meet new standards. Introduced the concept of RC versions for Aleph. Introduced Feature and Bugfix forms for easier issue creation. Bumped numerous libraries.
-- Fixed bug where you could hit an infinite scroll if the API returned inconsistent results.
-
-## 3.12.4 (2022-06-13)
-
-- The histogram feature used for dates now allows you to zoom in and out to get a better view of the dates that are important to you. Note that histograms aren't yet great with vague dates (January 2021, 1984) so dates with no day or month will default to the first day of the month, year.
-- Update pyjwt requirement from <2.4.0,>=2.0.1 to >=2.0.1,<2.5.0
-- Bump react-ftm
-- Bump followthemoney from 2.9.4 to 2.9.5
-- Bump @formatjs/intl-relativetimeformat from 10.0.1 to 11.0.1
-- Bump followthemoney from 2.9.3 to 2.9.4
-- Bump @alephdata/followthemoney from 2.9.3 to 2.9.4 in /ui
-- Bump alembic from 1.7.7 to 1.8.0 (#2296)
-- Bump jsonschema from 4.5.1 to 4.6.0 (#2300)
-- Bump sqlalchemy from 1.4.36 to 1.4.37 (#2297)
-
-## 3.12.3 (2022-06-01)
-
-- Fixed issue with Network diagrams where creating a Trip would cause the diagram to break
-- Fixed an issue where we were not concatenating error messages for the same path
-- Numerous library version bumps
-
-## 3.12.2 (2022-03-28)
-
-- The Aleph UI is now available in French.
-- Fix search handler on Aleph homescreen
-- Upgrade followthemoney to 2.8.5
-- Upgrade followthemoney to 2.8.4 in UI
-- Upgrade alembic to 1.7.7
-- Upgrade urllib to 1.26.9
-- Upgrade sqlalchemy to 1.4.32
-- Upgrade normality to 2.3.1
-- Upgrade servicelayer to 1.19.0
-- Upgrade flask to 2.0.3
-
-## 3.12.1 (2022-02-15)
-
-- Fix bugs introduced in 3.12.0 ([#2115](https://github.com/alephdata/aleph/pull/2115), [#2112](https://github.com/alephdata/aleph/pull/2112))
-
-## 3.12.0 (2022-02-02)
-
-{% hint style="warning" %}
-This release of Aleph introduced a couple of bugs that could make an Aleph instance unusable. Please skip this version in favour of newer Aleph versions.
-{% endhint %}
-
-- New followthemoney-compare ML model for xref [(#1818)](https://github.com/alephdata/aleph/pull/1818)
-- UI tweaks for timelines. Timelines are no longer behind tester flag; now available to everyone.
-- Users without write-access can see a read-only view of a dataset's source documents hierarchy
-- Fix Google OAuth integration issue [(#2062)](https://github.com/alephdata/aleph/pull/2062)
-- Upgrade to Elasticsearch 7.16.1 [(#2080)](https://github.com/alephdata/aleph/pull/2080)
-- Dependency upgrades and bug fixes
-- Upgrade ingest-file and convert-document to 3.16.1
- - Includes bug fix for PST mailbox processing [(ingest-file#197)](https://github.com/alephdata/ingest-file/pull/197)
- - Support for OCR in Khmer language [(ingest-file#194)](https://github.com/alephdata/ingest-file/pull/194)
-
-## 3.11.1
-
-- Dependency upgrades and bug fixes
-
-## 3.11.0
-
-- Allows users to configure custom facets for entity search screens - supports both type group facets (names, file types, addresses, etc.) as well as property facets.
-- Allows users to configure the columns in the search results screen
-- Adds NER support for Norwegian, Dutch and Danish languages
-- Dependency upgrades and bug fixes
-
-## 3.10.5
-
-- Stability and UX improvement in processing of mentions in large PDF files
-- Dependency upgrades
-- Logging improvement in ingest-file for better debugging of configuration issues
-
-## 3.10.4
-
-- Dependency upgrades
-
-## 3.10.2
-
-- Build support for arm64 so that Aleph can run on the new M1 Macbooks
-- New article viewer UI to show FtM Article entities
-- Dependency upgrades
-
-## 3.10.1
-
-- **TESTING:** New timeline editor for investigations, allows users to create and browse events.
-- Bug fixes and dependency upgrades.
-
-## 3.10.0
-
-- Fixed a bug in the tokenisation of the search index that dropped numbers from being made searchable. This has been fixed, but it only applies to collections (re-)indexed after this release.
-- Improved scoring in cross-references based on a regression model derived from user judgements. Also tuned the way Aleph compares properties in the "Mentions" tab of documents etc.
-- For Outlook email files (.msg), the RTF variant of the body will now be indexed in the form of an attachment to the message, titled `body.rtf`
-
-## 3.9.10
-
-- Inline the helm chart into the Aleph repository, it's now shipped with the main application. This requires updating your helm configuration if you've been using the previous charts.
-- Loads of bug fixes for small UI issues.
-
-## 3.9.9
-
-- Re-design the Investigation UI for a UX that involves guiding the user through some common actions.
-- Refactor much of the state handling in the React app.
-- Bug fixes on ingestors.
-- Allow entities in one collection to reference those in another.
-
-## 3.9.8
-
-- Re-name personal datasets to "Investigations" in the UI
-- Introduce user interfaces for profiles, an interactive way to de-duplicate data. Fix various bugs in profile logic in the backend.
-- Get rid of the global scoped search, show separate search bars closer to the subject of the search in the user interface.
-- Introduce structured logging of JSON objects in Stackdriver.
-- Polish data loading in the user interface and de-bug various features.
-
-## 3.9.7
-
-- Work on Arabic/RTL i18n, nested directionality.
-
-## 3.9.6
-
-- Debug OIDC logout
-- Pairwise judgement API to replace xref decisions API.
-
-## 3.9.5
-
-{% hint style="warning" %}
-In this version, the **OAuth configuration was changed in potentially breaking ways**. Please read the instructions below for how to adapt your deployment.
-{% endhint %}
-
-Aleph 3.9.5 uses [OpenID Connect](https://openid.net/connect/) to largely automate the configuration of delegated login. Previous versions of Aleph configured an OAuth2 client explicitly, which also required coding custom handlers for each OAuth provider. The new system also addresses a number of potential security issues.
-
-Unfortunately, the transition requires some incompatible changes:
-
-- You now need to configure a `ALEPH_OAUTH_METADATA_URL` to set an endpoint used by OIDC to self-configure.
- - Examples of valid metadata URLs for services like Google, Azure, Amazing Cognito and Keycloak can be found in the file `aleph.env.tmpl`.
- - The existing options `ALEPH_OAUTH_BASE_URL`, `ALEPH_OAUTH_TOKEN_URL` and `ALEPH_OAUTH_AUTHORIZE_URL` are no longer needed.
- - `ALEPH_OAUTH_HANDLER` and `ALEPH_OAUTH_SCOPE` are now optional.
-- The database IDs generated for users and **groups will be different**. For users, the ID should be re-written the first time a user logs in after the upgrade. Groups, on the other hand, may require a SQL intervention to adapt their IDs. For example, with a Keycloak provider, the change would be:`UPDATE role SET foreign_id = REPLACE(foreign_id, 'kc:', 'group:') WHERE type = 'group';`
-
-Beyond these breaking changes, some other differences are notable:
-
-- Logging out of Aleph will now also log a user out of the OAuth provider, where supported (e.g. Keycloak, Azure).
-- If a user is blocked or deleted while using the site, their session will be disabled by the worker backend within an hour. (This can be forced by running `aleph update`)
-
-Changes unrelated to OAuth:
-
-- EntitySets no longer contain an `entities` array of all their members. Use the sub-resource `/entitysets/x/entities` instead.
-- Multiple bug fixes in UI related to i18n.
-
-## 3.9.4
-
-- Move file ingestor service `ingest-file` to its own repository to decouple versioning and CI/CD.
-
-## 3.9.3
-
-- Show transliterated names of non-latin entities in the user interface.
-- Refactor query serialisation, remove in-database query log.
-- Fix out of memory errors in cross-reference
-- Extensive bug fixes in mapping UI
-
-## 3.9.1
-
-- Data exports feature to let users make offline data exports for searches and cross-reference
-- New home page, based on the stupid CMS we introduced for the about section.
-- Ability to map entities into lists via the UI and alephclient.
-- Tons of bug fixes in UI and backend.
-
-## 3.9.0
-
-- UI for managing lists of entities within a dataset. This lets you make sub-sets of a dataset, e.g. "The Family", "Lawyers" or "Core companies".
-- Ability to cross-reference a collection of documents against structured data collections using `Mention` schema stubs. Requires dataset reingest before it takes effect.
-- New internationalisation mechanism for the React bits, using JSON-formatted translation files.
-
-## 3.8.9
-
-- Move the linkages API ("god entities" / record linkage) to use entity sets instead of its own database model.
-- Remove soft-deletion for some model types (permissions, entities, alerts, mappings).
-
-{% hint style="danger" %}
-Aleph 3.8.9 combines all database migrations before Aleph 3.2 into a single version. If you want to upgrade from an Aleph older than 3.2, we recommend you move via 3.8.0, upgrade to that version, before migrating across this version.
-{% endhint %}
-
-## 3.8.6
-
-- Date histogram facet and filtering tool on search results.
-- Added example code for how to [add text processors](/developers/adding-text-processors) to Aleph.
-- Re-worked collection stats caching to avoid super slow requests when no cache is present.
-- Tons of bug fixes.
-
-## 3.8.5
-
-- Introduce EntitySets, as user-curated sets of ... entities! All diagrams are now entitysets, as will be timelines and bookmarks.
-
-## 3.8.3
-
-- Refactor queue and processing code for the Aleph worker.
-
-## 3.8.1
-
-- "Expand node" support in network diagrams pulls relevant connections from the backend and shows them to the user while browsing a network diagram.
-- Correctly handle the use of multi-threading when using Google Cloud Storage Python client libraries.
-
-## 3.8.0
-
-- We've re-worked the way entities are aggregated before they are being loaded into the search index. This was required because Aleph is become more interactive and needs to handle non-bulk operations better. It also improves metadata handling, like which user uploaded a document, or when an entity was last updated. Aleph will now always keep a full record of the entities in the SQL database, whichever way they are submitted. To this end, we've migrated from `balkhash` to `followthemoney-store` (i.e. balkhash 2.0). This will start to apply to existing collections when they are re-ingested or re-indexed.
-- Aleph has two new APIs for doing a collection `reingest` and `reindex`. The existing `process` collection API is gone. `alephclient` now supports running `reingest`, `reindex`, and `delete` on a collection.
-- Operators can expedite the rollout of the new backend by running `aleph reingest-casefiles` and `aleph reindex-casefiles` to re-process all existing personal datasets.
-- Numerous UI fixes make the table editor and network diagrams much more smooth.
-
-## 3.7.2
-
-- We've introduced a table editor in the user interface for manually editing entities in personal datasets.
-
-## 3.7.0
-
-- A graph expand API for entities returns all entities adjacent to an entity for network-based exploration of the data.
-
-## 3.6.4
-
-- **Linkages**, a new data model. A linkage is essentially an annotation on an entity saying it is the same as some other entities (in other datasets). This would, for example, let you group together all mentions of a politician into a single profile. Linkages are currently created via the Xref UI, which now has a ‘review mode’.
-- In the future, profiles (ie. the composite of many linkages) will start showing up in the UI in different places, to introduce an increasingly stronger notion of data integration. Because linkages are based on a reporter’s judgement, they belong to either a) them, or b) a group of users — so they are always a bit contextualised, not fully public.
-- Our hope is also that the data collected via linkages will provide training material for a machine learning-based approach to cross-referencing.
-
-## 3.6.3
-
-- Users who employ OAuth may need to change their settings to define a `ALEPH_OAUTH_HANDLER` in their `aleph.env` . By default, the following handlers are supported: `google`, `keycloak`, `azure`.
-
-## 3.5.0
-
-- Run VIS2 / [Network diagrams](/guide/building-out-your-investigation/network-diagrams) on Aleph as a testing feature.
-
-## 3.4.9
-
-- Two SECURITY ISSUES in the software: one that would let an attacker enumerate registered users, and the other could be exploited for XSS with a forged document. They were discovered by two friendly hackers from blbec.online who kindly reported them to us.
-
-## 3.4.0
-
-- The **mapping UI**. Prototyped by [@Felix Ebert](https://alephdata.slack.com/team/UE32DAC4S), [@Kirk](https://alephdata.slack.com/team/UL1AWH89X) and [@sunu](https://alephdata.slack.com/team/UE1EFLX5K) have done great work on this. The idea is that for a simple CSV file you can just upload it, and use the UI to map it into entities that you can cross-reference. It’s really simple to use and useful.
-- `synonames`. This is an extension to our install of ElasticSearch that allows us to expand names into cultural transliterations. So for example doing a search for `Christoph` will now also search `Кристоф`, even though they aren’t literally the same names. This should increase recall for cross-cultural queries. The whole thing was a project from [@Aparna](https://alephdata.slack.com/team/UML9VA9K5), generating these aliases from Wikidata entries.
-- These changes come alongside a lot of UI and backend polishing, so things should be much more smooth all around.
-
-## 3.0.0
-
-The goal of `aleph` 3.0.0 is to harmonise the handling of data inside the index. Instead of having different formats and mappings for documents, entities, table rows and document pages, there is now just one type of index object: an entity.
-
-This means that document-based data is now completely 'translated' to the `followthemoney` ontology used by `aleph` (meaning that in theory, each page of a document and each row of a table is now a node in the object graph of the `aleph` platform).
-
-### Upgrading
-
-In order to accomplish this, a complete re-index is required in all cases. The recommended path of migrating from a 2.x.x installation is this set of commands in an aleph container shell (`make shell`):
-
-```bash
-# Re-create the indexes:
-aleph resetindex
-# Apply a database schema change:
-aleph upgrade
-# Re-index collections and documents:
-aleph repair --entities
-```
-
-Be advised that any data loaded via the entity mapping mechanism will need to be re-loaded after this. It is also worth noting that at OCCRP, we have now started generating mapped data via the `followthemoney` command-line tool, and are using `alephclient` to bulk-load the resulting stream of entities into the system. This has proven to be significantly quicker than the built-in mapping process.
-
-### Other changes
-
-- Settings `ALEPH_REDIS_URL` and `ALEPH_REDIS_EXPIRE` are now `REDIS_URL` and
-
- `REDIS_EXPIRE`.
-
-- Variable `ALEPH_OCR_VISION_API` is now `OCR_VISION_API`, it will enable use of
-
- the Google Vision API for optical character recognition.
-
-- The `/api/2/collections//ingest` API now only accepts a single file, or
-
- no file (which will create a folder). The response body contains only the ID
-
- of the generated document. The status code on success is now 201, not 200.
diff --git a/docs/src/pages/developers/datacommons.mdx b/docs/src/pages/developers/datacommons.mdx
index dacd38c792..02853377a7 100644
--- a/docs/src/pages/developers/datacommons.mdx
+++ b/docs/src/pages/developers/datacommons.mdx
@@ -1,5 +1,5 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: Data commons
---
@@ -13,9 +13,9 @@ The data below is available in [Follow the Money](/developers/followthemoney/) f
## OpenSanctions
-Sanctions lists, databases of politicians and criminal figures can be a good way to find leads within other data, using [cross-referencing](/guide/building-out-your-investigation/cross-referencing). The [OpenSanctions](http://opensanctions.org) project generates consolidated sanctions data from multiple sources, and provides the entries as Follow the Money \(FtM\) data.
+Sanctions lists, databases of politicians and criminal figures can be a good way to find leads within other data, using [cross-referencing](/users/investigations/cross-referencing). The [OpenSanctions](http://opensanctions.org) project generates consolidated sanctions data from multiple sources, and provides the entries as Follow the Money \(FtM\) data.
-**See:** [**https://opensanctions.org**](https://opensanctions.org)
+**See: [https://opensanctions.org](https://opensanctions.org)**
## ICIJ OffshoreLeaks
diff --git a/docs/src/pages/developers/developer-tools.mdx b/docs/src/pages/developers/developer-tools.mdx
index 9b168f2ad3..049c3ca9fa 100644
--- a/docs/src/pages/developers/developer-tools.mdx
+++ b/docs/src/pages/developers/developer-tools.mdx
@@ -1,5 +1,5 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: Developer tools
---
diff --git a/docs/src/pages/developers/followthemoney/ftm.mdx b/docs/src/pages/developers/followthemoney/ftm.mdx
index dbfbf4b752..b65b6f51e7 100644
--- a/docs/src/pages/developers/followthemoney/ftm.mdx
+++ b/docs/src/pages/developers/followthemoney/ftm.mdx
@@ -1,5 +1,5 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: Command-Line
---
diff --git a/docs/src/pages/developers/followthemoney/index.mdx b/docs/src/pages/developers/followthemoney/index.mdx
index a7120427a6..f7f692c00e 100644
--- a/docs/src/pages/developers/followthemoney/index.mdx
+++ b/docs/src/pages/developers/followthemoney/index.mdx
@@ -1,5 +1,5 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: FollowTheMoney
---
diff --git a/docs/src/pages/developers/intro.mdx b/docs/src/pages/developers/index.mdx
similarity index 90%
rename from docs/src/pages/developers/intro.mdx
rename to docs/src/pages/developers/index.mdx
index 5e08cd5cda..3441896b1d 100644
--- a/docs/src/pages/developers/intro.mdx
+++ b/docs/src/pages/developers/index.mdx
@@ -1,13 +1,11 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: Technical introduction
---
-# Technical introduction
+# Technical Documentation
-
- Aleph is a toolkit of powerful components for processing knowledge graphs, focussed around the Aleph API server and document processing framework.
-
+
Welcome to the Aleph Technical Documentation. The tech docs contain resources for developers, administrators, and data engineers.
Aleph is an open source toolkit for investigative data analysis. It allows generating, searching and analysing large graphs of heterogeneous data, including public records, structured databases and leaked evidence. The system can integrate data from both unstructured data formats \(like PDF, Email, and other file types\) and structured data such as CSV files, or SQL databases. Data that's been loaded can be securely searched, cross-referenced with other datasets and exported to other systems.
diff --git a/docs/src/pages/developers/installation.mdx b/docs/src/pages/developers/installation.mdx
index 75dd4d4159..5f1d6890b7 100644
--- a/docs/src/pages/developers/installation.mdx
+++ b/docs/src/pages/developers/installation.mdx
@@ -1,5 +1,5 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: Installing Aleph
---
@@ -154,13 +154,29 @@ To build the image you can run `make build`, which will build the `alephdata/ale
### Inspecting ElasticSearch
-At times it can be useful to inspect the contents of the ElasticSearch indices. The developer setup includes [dejavu](https://github.com/appbaseio/dejavu), a web-based UI for ElasticSearch. In a local environment brought up as described above you should see dejavu under [http://localhost:1358](http://localhost:1358).
+At times it can be useful to inspect the contents of the ElasticSearch indices. There are several ElasticSearch GUIs out there that you can use to query ElasticSearch. You could for example use [Dejavu](https://github.com/appbaseio/dejavu):
-You will need to pass it the local ES cluster URL (`http://localhost:19200`). You can either inspect an individual index or use `*` to see the whole content. Index names for entity indices follow the pattern `aleph-entity-{{SCHEMA_NAME}}-{{VERSION}}`. For example, the index name for `LegalEntity` entities is `aleph-entity-legalentity-v1`.
+```bash
+docker run -p 1358:1358 -d appbaseio/dejavu
+open http://localhost:1358/
+```
+
+Then you can connect Dejavu to the ElasticSearch container that has been exposed on `http://localhost:9200`.
+
+You can also [approach ElasticSearch from the command line](https://www.elastic.co/guide/en/cloud/current/ec-working-with-elasticsearch.html).
### Inspecting the PostgreSQL database
-Run `make shell-db` to be dropped into a [psql](https://www.postgresql.org/docs/current/app-psql.html) shell within the database container.
+If you want to inspect or manipulate the SQL database directly \(e.g. to edit a user, create or delete a group\), you can connect to the PostgreSQL database.
+
+In development mode, the database is exposed on the host at `127.0.0.1:5432`. \(User, password and database name are all `aleph`\). You can also connect from the shell container:
+
+```bash
+make shell
+psql $ALEPH_DATABASE_URI
+```
+
+The same can be done if you run an instance of the `shell` container in production mode.
## Production deployment
diff --git a/docs/src/pages/developers/mappings.mdx b/docs/src/pages/developers/mappings.mdx
index febcaa0c42..c1ba219b98 100644
--- a/docs/src/pages/developers/mappings.mdx
+++ b/docs/src/pages/developers/mappings.mdx
@@ -1,5 +1,5 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: Importing structured data
---
diff --git a/docs/src/pages/developers/memorious.mdx b/docs/src/pages/developers/memorious.mdx
index b11c739911..c30476caa9 100644
--- a/docs/src/pages/developers/memorious.mdx
+++ b/docs/src/pages/developers/memorious.mdx
@@ -1,5 +1,5 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: Crawling data with Memorious
---
diff --git a/docs/src/pages/developers/mixed-graphs.mdx b/docs/src/pages/developers/mixed-graphs.mdx
index a4efb49c8d..b9c78d64f6 100644
--- a/docs/src/pages/developers/mixed-graphs.mdx
+++ b/docs/src/pages/developers/mixed-graphs.mdx
@@ -1,5 +1,5 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: Mixed document/entity graphs
---
diff --git a/docs/src/pages/developers/technical-faq/design-premises.mdx b/docs/src/pages/developers/technical-faq/design-premises.mdx
index e13da8ecb2..f1367864aa 100644
--- a/docs/src/pages/developers/technical-faq/design-premises.mdx
+++ b/docs/src/pages/developers/technical-faq/design-premises.mdx
@@ -1,5 +1,5 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: Design premises
---
diff --git a/docs/src/pages/developers/technical-faq/index.mdx b/docs/src/pages/developers/technical-faq/index.mdx
index 4549d7d315..f26d6a0450 100644
--- a/docs/src/pages/developers/technical-faq/index.mdx
+++ b/docs/src/pages/developers/technical-faq/index.mdx
@@ -1,5 +1,5 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: Technical FAQ
---
@@ -72,7 +72,7 @@ Here's a guide for [running Aleph sans docker on Debian w/ systemd](look-ma-no-d
Aleph does not perform updates and database migrations automatically. Once you have the latest version, you can run the command bellow to upgrade the existing installation \(i.e. apply changes to the database model or the search index format\).
-Before you upgrade, check the [release notes](../changelog) to make sure you understand the latest release and know about new options and features that have been added.
+Before you upgrade, check the [release notes](https://github.com/alephdata/aleph/releases) to make sure you understand the latest release and know about new options and features that have been added.
The procedures for upgrading are different between production and development mode:
@@ -243,19 +243,6 @@ You may also need to run `aleph update` afterwards to refresh some cached inform
That's where it's most at home! We recommend you use the [helm chart](https://github.com/alephdata/aleph/tree/main/helm) to deploy Aleph. It will allow you to override the key settings for your site, while providing a coherent deployment. We use auto-scaling both on the cluster and pod level, which helps to combine fast imports with limited operational cost.
-## How can I connect to the database directly?
-
-If you want to manipulate the SQL database directly \(e.g. to edit a user, create or delete a group\), you can connect to the PostgreSQL database.
-
-In development mode, the database is exposed on the host at `127.0.0.1:15432`. \(User, password and database name are all `aleph`\). You can also connect from the shell container:
-
-```bash
-make shell
-psql $ALEPH_DATABASE_URI
-```
-
-The same can be done if you run an instance of the `shell` container in production mode.
-
## Why do entities have two-part IDs?
When looking at an Aleph URL, you may notice that every entity ID has two parts, separated by a dot \(`.`\), for example:`deadbeef.3cd336a9859bdf2be917f561430f2a83e5da292b`. The first part in this is the actual entity ID, while the second part is a signature \(HMAC\) assigned by the server when indexing the data.
diff --git a/docs/src/pages/developers/technical-faq/look-ma-no-docker.mdx b/docs/src/pages/developers/technical-faq/look-ma-no-docker.mdx
index 81c0a6ad14..8bf76a37ea 100644
--- a/docs/src/pages/developers/technical-faq/look-ma-no-docker.mdx
+++ b/docs/src/pages/developers/technical-faq/look-ma-no-docker.mdx
@@ -1,5 +1,5 @@
---
-layout: '@layouts/DocsLayout.astro'
+layout: '@layouts/DevelopersLayout.astro'
title: Look Ma, No Docker
---
diff --git a/docs/src/pages/get-in-touch.mdx b/docs/src/pages/get-in-touch.mdx
deleted file mode 100644
index 2e89796e08..0000000000
--- a/docs/src/pages/get-in-touch.mdx
+++ /dev/null
@@ -1,25 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Get in touch
----
-
-# Get in touch
-
-
- Aleph is an open source project developed by staff and volunteers. Everyone is invited to participate in the community, ask questions and contribute to the software.
-
-
-As an open source community, we welcome anyone who wants to use and improve Aleph. Participants are required to **be excellent to each other**.
-
-
- While the Aleph source code is openly available and licensed, the community
- support for the system is subject to some criteria. [Read
- more](https://github.com/alephdata/aleph/blob/main/SUPPORT.md)
-
-
-- Most discussion and support takes place on the **Aleph Slack**. Please follow the steps lined out in the [Aleph Support Policy](https://github.com/alephdata/aleph/blob/main/SUPPORT.md) to receive an invitation for the community.
-- If you notice a **defect with the software**, head on over to [GitHub to file an issue report](https://github.com/alephdata/aleph/issues). Remember to include information about what you were trying to accomplish, and what specific operating system you are using.
-- For questions related to the instance of Aleph operated by the [Organized Crime and Corruption Reporting Project](https://data.occrp.org) (OCCRP), contact the **data team at data@occrp.org**. Our data team provides assistance to individuals and groups engaged in public interest journalism. We do not currently offer commercial support and consultancy related to Aleph.
-- **If you are a whistleblower** and wish to share confidential information with the press, this community of software developers is not the ideal place to do so. [Contact a journalistic organisation](https://www.occrp.org/en/become-a-whistleblower/) instead.
-
-
diff --git a/docs/src/pages/get-in-touch/index.mdx b/docs/src/pages/get-in-touch/index.mdx
new file mode 100644
index 0000000000..da083848d6
--- /dev/null
+++ b/docs/src/pages/get-in-touch/index.mdx
@@ -0,0 +1,28 @@
+---
+layout: '@layouts/DefaultLayout.astro'
+title: Get in touch
+---
+
+# Get in touch
+
+Aleph is an open source project developed by staff and volunteers. Everyone is invited to participate in the community, ask questions and contribute to the software.
+
+
+ While the Aleph source code is openly available and licensed, the community support for the system is subject to some criteria. Please refer to our [Support Policy](https://github.com/alephdata/aleph/blob/main/SUPPORT.md).
+
+
+## Slack Community
+
+Most discussion and support takes place on the **Aleph Slack**. Please follow the steps lined out in the [Aleph Support Policy](https://github.com/alephdata/aleph/blob/main/SUPPORT.md) to receive an invitation for the community.
+
+## Bugs
+
+If you notice a **defect with the software**, head on over to [GitHub to file an issue report](https://github.com/alephdata/aleph/issues). Remember to include information about what you were trying to accomplish, and what specific operating system you are using.
+
+## OCCRP Aleph Support
+
+For questions related to the instance of Aleph operated by the [Organized Crime and Corruption Reporting Project](https://data.occrp.org) (OCCRP), contact the data team using [this form](https://requests.occrp.org/data). Our data team provides assistance to individuals and groups engaged in public interest journalism. We do not currently offer commercial support and consultancy related to Aleph.
+
+## Whistleblowers
+
+**If you are a whistleblower** and wish to share confidential information with the press, this community of software developers is not the ideal place to do so. [Contact a journalistic organisation](https://www.occrp.org/en/become-a-whistleblower/) instead.
diff --git a/docs/src/pages/guide/.DS_Store b/docs/src/pages/guide/.DS_Store
deleted file mode 100644
index 8d25965dc2..0000000000
Binary files a/docs/src/pages/guide/.DS_Store and /dev/null differ
diff --git a/docs/src/pages/guide/aleph-data-desktop.mdx b/docs/src/pages/guide/aleph-data-desktop.mdx
deleted file mode 100644
index cf23eca5a5..0000000000
--- a/docs/src/pages/guide/aleph-data-desktop.mdx
+++ /dev/null
@@ -1,72 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Aleph Data Desktop
----
-
-# Aleph Data Desktop
-
-
- **September 2022:** Aleph Data Desktop has been deprecated and will not receive any further updates. All of Data Desktop’s features are also available within Aleph. Please refer to the Aleph documentation to find out how to [create network diagrams](/guide/building-out-your-investigation/network-diagrams) in Aleph and how to [import diagrams](#importing-network-diagrams-into-aleph-from-aleph-data-desktop) created in Data Desktop.
-
-
-[Creating network diagrams ](/guide/building-out-your-investigation/network-diagrams)within Aleph allows you to map networks of people and influence directly alongside the rich ecosystem of data and processing tools that Aleph provides. Access control ensures that the investigation files that you create are accessible only to collaborators with which you explicitly share them.
-
-Even with these protections, there might arise a scenario where you want to **work on an investigation file offline**, within the private confines of your own device. For this reason, we have released an application called Aleph Data Desktop.
-
-Follow the steps below to download Aleph Data Desktop and to learn how to move diagrams between the desktop application and Aleph.
-
-## Download the Desktop App
-
-Aleph Data Desktop is available for download for [**Microsoft Windows**](https://github.com/alephdata/datadesktop/releases/latest/download/Aleph-Data-Desktop.exe), [**Apple MacOS X**](https://github.com/alephdata/datadesktop/releases/latest/download/Aleph-Data-Desktop.dmg)**,** and common [**Linux distributions**](https://github.com/alephdata/datadesktop/releases/latest/download/Aleph.Data.Desktop.deb). Updates to the application are released periodically, and most users will be prompted with the option to download the latest release when their current version becomes out-of-date.
-
-If you are interested, you are also welcome to explore and contribute to [the source code](https://github.com/alephdata/visdesktop) of the application.
-
-## Using Aleph Data Desktop
-
-Using the desktop app, investigation files can be created, edited, and shared **as you would any other file on your computer**, and will not be shared with others unless you explicitly decide to send that file to them.
-
-[Editing diagrams](/guide/building-out-your-investigation/network-diagrams) or [using the table editor](/guide/building-out-your-investigation/using-the-table-editor) within the desktop app works exactly the same as in Aleph. The only difference is that, in the desktop application, you do not have the ability to enrich and expand entities in your diagram with additional linkages found in the Aleph ecosystem.
-
-Another important note is that, unlike in Aleph's Network Diagrams editor where changes are saved automatically, in Aleph Data Desktop you must click **"Save"** in the menu bar to save changes to your diagram file.
-
-
-
-## Exporting network diagrams from Aleph to Aleph Data Desktop
-
-A network diagram created in Aleph can easily be exported, and then subsequently edited, in Aleph Data Desktop. It is important to note, however, that any changes made in the desktop application will not be reflected in the original version of your diagram in Aleph.
-
-To export a network diagram from Aleph, click the **"Export"** button in the top right corner when viewing any diagram.
-
-
-
-A file called _your_diagram_name_.vis will now be downloaded to your computer.
-
-
-
-If you have Aleph Data Desktop downloaded, clicking this file should automatically open your diagram in the desktop application. Alternatively, with the application open, click the **"Open"** button in the top menu bar, and then select your file from your computer's file system.
-
-
-
-## Importing network diagrams into Aleph from Aleph Data Desktop
-
-There might be occasions where is useful to import a diagram you have been working on in the desktop application into Aleph.
-
-
- Save your diagram file.
- In an Aleph investigation, go to the **"Network diagrams"** tab.
-
- Then click the **"Import diagram"** button.
-
-
-
-
- **Drag** your diagram file into the window, or click to select it from your computer's file system.
-
-
-
-
- Modify the title and description as you wish. Then click **"Create"**.
-
- Your diagram will now be visible and editable within Aleph, and any entities and relationships contained within it will automatically be added to the investigation into which it was added.
-
-
diff --git a/docs/src/pages/guide/building-out-your-investigation/creating-a-personal-dataset.mdx b/docs/src/pages/guide/building-out-your-investigation/creating-a-personal-dataset.mdx
deleted file mode 100644
index a536378ae9..0000000000
--- a/docs/src/pages/guide/building-out-your-investigation/creating-a-personal-dataset.mdx
+++ /dev/null
@@ -1,39 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Creating a personal dataset
----
-
-# Creating a personal dataset
-
-
- The start of any investigation in Aleph is a Personal dataset. Read below to find out how to create, manage, and share a workspace for your investigation.
-
-
-Personal datasets are **contained workspaces** within Aleph where you can **upload**, **edit**, and **organize data** related to an investigation or topic of interest - and they can be shared with any other user or access group within Aleph.
-
-While source datasets, like company registries and other evidentiary collections, cannot be edited, Personal datasets allow you to edit the contents by [uploading documents](/guide/building-out-your-investigation/uploading-documents), [mapping data into structured entities](/guide/building-out-your-investigation/generating-multiple-entities-from-a-list), and [creating your own network diagrams](/guide/building-out-your-investigation/network-diagrams).
-
-To create a Personal dataset, click the "**Personal datasets"** button in the left sidebar on the homepage and then click the **"New dataset"** button.
-
-
-
-## Managing Access to your Personal Dataset
-
-When you create a Personal dataset, its contents are by default accessible to **you and no one else**. But access can easily be expanded to any co-collaborators you would like to include in your investigation.
-
-To grant colleagues access to your dataset, click the **"Share"** button in the top right toolbar on your dataset's homepage.
-
-
-
-The access control window will then appear, allowing you to share your Personal dataset with **other individuals** or **groups** you are a part of.
-
-An individual must first have a user account on Aleph to receive access to a dataset. Sharing a dataset with a group means that any user in that group will receive the access you are granting.
-
-- **View access** allows a user to browse the contents of a Personal dataset but not create, edit, or delete anything contained within
-- **Edit access** allows a user full privileges to modify its contents
-
-
-
-_Note: only Aleph administrators have the ability to make a Personal dataset publicly available._
-
-**Continue on to find out more about what you can do now that you've created a Personal dataset for your investigation.**
diff --git a/docs/src/pages/guide/building-out-your-investigation/creating-an-investigation.mdx b/docs/src/pages/guide/building-out-your-investigation/creating-an-investigation.mdx
deleted file mode 100644
index 98ea29d5cd..0000000000
--- a/docs/src/pages/guide/building-out-your-investigation/creating-an-investigation.mdx
+++ /dev/null
@@ -1,39 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Creating an investigation
----
-
-# Creating an investigation
-
-
- Read below to find out how to create, manage, and share a workspace for your investigation.
-
-
-Investigations are **contained workspaces** within Aleph where you can **upload**, **edit**, and **organize data** related to an investigation or topic of interest - and they can be shared with any other user or access group within Aleph.
-
-While source datasets, like company registries and other evidentiary collections, cannot be edited, investigations allow you to edit the contents by [uploading documents](/guide/building-out-your-investigation/uploading-documents), [mapping data into structured entities](/guide/building-out-your-investigation/generating-multiple-entities-from-a-list), and [creating your own network diagrams](/guide/building-out-your-investigation/network-diagrams).
-
-To create an investigation, click the "**Investigations"** button in the left sidebar on the homepage and then click the **"New investigation"** button.
-
-
-
-## Managing access to your investigation
-
-When you create an investigation, its contents are by default accessible to **you and no one else**. But access can easily be expanded to any co-collaborators you would like to include in your investigation.
-
-To grant colleagues access to your investigation, click the settings button in the top right toolbar on your investigation homepage.
-
-
-
-The access control window will then appear, allowing you to share your investigation with **other individuals** or **groups** you are a part of.
-
-An individual must first have a user account on Aleph to receive access to an investigation. Sharing an investigation with a group means that any user in that group will receive the access you are granting.
-
-- **View access** allows a user to browse the contents of an investigation but not create, edit, or delete anything contained within
-- **Edit access** allows a user full privileges to modify its contents
-
-
-
-Only Aleph administrators have the ability to make an investigation publicly available.
-
-**Continue on to find out more about what you can do now that you've created an investigation.**
diff --git a/docs/src/pages/guide/building-out-your-investigation/cross-referencing.mdx b/docs/src/pages/guide/building-out-your-investigation/cross-referencing.mdx
deleted file mode 100644
index aed7290fc6..0000000000
--- a/docs/src/pages/guide/building-out-your-investigation/cross-referencing.mdx
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Cross-referencing
----
-
-# Cross-referencing
-
-
- Aleph can conduct bulk searches to find all mentions of companies or people in one data source to those in all others. This can be a useful method to generate reporting leads.
-
-
-Cross-referencing is a powerful means for reporters to do mass comparisons between persons of interest, public records and leaked data. See below for a tutorial presentation on the subject:
-
-
-
-
diff --git a/docs/src/pages/guide/building-out-your-investigation/generating-multiple-entities-from-a-list.mdx b/docs/src/pages/guide/building-out-your-investigation/generating-multiple-entities-from-a-list.mdx
deleted file mode 100644
index 4a20fb2a7b..0000000000
--- a/docs/src/pages/guide/building-out-your-investigation/generating-multiple-entities-from-a-list.mdx
+++ /dev/null
@@ -1,63 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Generating multiple entities from a spreadsheet or CSV
----
-
-# Generating multiple entities from a spreadsheet or CSV
-
-
- Follow the steps below to extract structured entities from an uploaded spreadsheet within an investigation.
-
-
-When **importing a list of companies, people or similar data objects**, it is helpful to convert your raw data into [structured entities](/developers/followthemoney/) that Aleph can use to classify those entities for searching, cross-referencing, and many other operations.
-
-
- For small and mid-size datasets, follow the instructions below to map your data into structured entities using the Aleph mapping editor. To map very large data straight from a SQL database, or to map complicated relationships, you will have to [create a mapping file](/developers/mappings) instead.
-
-
-
- Follow the "Importing documents and files" steps above to **import your .csv, .xls, or .xlsx data file** to an investigation.
-
- Once the data file has finished uploading, browse to one of the tables you have uploaded. C**lick the "Generate Entities" tab** to open the mapping editor.
-
-
-
-
- **Select the types** of entities you would like to create from your data
-
-
-
-
- **Define any relationships** - such as ownership or family relations - that exist between the entities.
-
-
-
-
- **Select columns from your data to use as unique keys** for each entity.
-
- Keys are usually ID numbers, email addresses, telephone numbers, or other properties of your data which are guaranteed to be unique. The more columns you are able to select as keys, the better, to ensure that Aleph correctly generates an entity for each unique entry in your data.
-
-
-
-
- **Assign data columns to properties** of the entity types you have selected.
-
-
-
-
- **Add any fixed values to the entities.** If, for instance, your data is a company registry from Moldova, it would be useful to set the "Country" property to "Moldova", so that all company entities that are generated will automatically have their country set as Moldova.
-
-
-
-
- **Select an optional destination for the entities that are created**. By selecting a diagram or list here, the entities that are generated from your spreadsheet will be automatically added to that diagram or list, in addition to the investigation of which they are a part.
-
-
-
-
- **Verify and submit your mapping.** When a mapping is submitted, Aleph will automatically generate structured entities based on the instructions you have provided it. It is important to note that **any** **future edits you make to the mapping will re-generate these entities.** Additionally, deleting the mapping will delete any entities generated from it.
-
-
- **After the entity generation process has finished, you should see your entities in the** [**entities view**](/guide/building-out-your-investigation/using-the-table-editor) **of your investigation.**
-
-
diff --git a/docs/src/pages/guide/building-out-your-investigation/index.mdx b/docs/src/pages/guide/building-out-your-investigation/index.mdx
deleted file mode 100644
index 6f80d2d642..0000000000
--- a/docs/src/pages/guide/building-out-your-investigation/index.mdx
+++ /dev/null
@@ -1,27 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Building out your investigation
----
-
-# Building out your investigation
-
-
- Aleph provides a powerful suite of tools to analyze your data, find
- connections across datasets and investigations, and share findings with collaborators. Browse the guides below to learn more.
-
-
-Managing the data of an investigation with Aleph not only allows you to access a growing collection of tools for processing and organizing your findings, but also enables you to connect your accumulated knowledge with hundreds of other datasets contained in Aleph.
-
-
-
-Once you have created an investigation, an array of tools are available to help you organize, structure, and analyze aspects of your investigation.
-
-
-
-
-
-
-
-
-
-
diff --git a/docs/src/pages/guide/building-out-your-investigation/network-diagrams.mdx b/docs/src/pages/guide/building-out-your-investigation/network-diagrams.mdx
deleted file mode 100644
index 7773f3c716..0000000000
--- a/docs/src/pages/guide/building-out-your-investigation/network-diagrams.mdx
+++ /dev/null
@@ -1,72 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Drawing network diagrams
----
-
-# Drawing network diagrams
-
-
- Over the course of an investigation, it is sometimes helpful to sketch out a network of involved parties so that you can understand who is linked to what.
-
-
-**Network diagrams** let you illustrate webs of people, companies and other entities involved in an investigation in a visual diagram of actors and connections. It utilizes the same [types of nodes and connections](/developers/followthemoney/) as the rest of Aleph. It is important to note that all entities and relationships that you add to your diagram will be automatically added to the investigation in which the diagram is contained.
-
-## Creating your first diagram
-
-Network diagrams can be created within any [**investigation**](/guide/building-out-your-investigation/creating-an-investigation) you have access to in Aleph. To get started, click the "**Network diagrams**" button in the sidebar on any investigation page, and then the "**New diagram**" button.
-
-
-
-You will now be able to map people, companies, and the relationships between them, which will be saved within your Aleph account, and only be visible to you and whomever you choose to share the diagram with.
-
-## Adding entities to your diagram
-
-To add people, companies, or other entities to your diagram, click the **"Add entities"** icon in the toolbar on the left side of the screen, or double click anywhere in the graphing area.
-
-
-
-Once you have created multiple entities in your diagram, you can connect them by clicking the **"Add links"** icon.
-
-
-
-Any entities and relationships you create within a diagram will automatically be added to the investigation in which your diagram was created. This allows you to cross-reference the lists of people, companies, and other entities you have created in your diagrams with data from across the rest of Aleph.
-
-## Sharing diagrams
-
-Once you have created a diagram, you can allow others to view or edit it by allowing them access to the investigation in which it is contained.
-
-You can also export your diagram to be saved as a file on your computer by clicking the **"Export"** button in the top right corner of the diagram editor. Once the diagram has been exported as a file, it can be shared with others as with any other file on your computer.
-
-## Frequently Asked Questions
-
-Below are some of the common questions we've received about network diagrams. If you have another question or want to suggest an improvement, please [get in touch](/get-in-touch).
-
-### How does this relate to OCCRP VIS?
-
-[Visual Investigative Scenarios](https://vis.occrp.org/) (VIS) is an application developed by OCCRP, [RISE Project](https://www.riseproject.ro/) and [QuickData](http://www.quickdata.ro/) that is the inspiration for network diagrams. While it's been a beloved tool to some reporters, it has also become slow and cumbersome.
-
-For this reason, we have made the difficult deicison to disable signups and the creation of new visualizations in VIS. Existing VIS diagrams stay accessible.
-
-If you’re looking for a visual mindmapping tool to use in you investigations, we recommend using network diagrams in Aleph instead.
-
-### How much data can I add to a network diagram?
-
-The goal of network diagrams in Aleph is to support investigative mind mapping, not big data analysis. While you may be able to add several hundred companies and people to a chart, the ideal size of a chart is somewhere around to 50-[150 entities](https://en.wikipedia.org/wiki/Dunbar's_number).
-
-### Can I make entities into groups?
-
-Yes. Simply select two or more entities, then click the "Group selection" button, as depicted below.
-
-
-
-### Can I embed a diagram in an article?
-
-Yes. In main the toolbar, you have an option to export an SVG graphic to a file. This SVG can be opened and "beautified" in a program like Adobe Illustrator or Inkscape (open source), and eventually published to the web as an image.
-
-Please be conscious, however, that network diagrams are mainly intended to be investigative sketches. Their complexity can easily exceed the complexity we should force our readers to deal with. Be nice to the mortals.
-
-### Can I import a spreadsheet of names and companies into a diagram?
-
-Yes! Follow the instructions for [generating multiple entities from a spreadsheet](/guide/building-out-your-investigation/generating-multiple-entities-from-a-list). In step 4 of the entity mapping process, select the desired destination diagram, and they will be automatically added to your diagram.
-
-
diff --git a/docs/src/pages/guide/building-out-your-investigation/uploading-documents.mdx b/docs/src/pages/guide/building-out-your-investigation/uploading-documents.mdx
deleted file mode 100644
index ed0eb7e7ce..0000000000
--- a/docs/src/pages/guide/building-out-your-investigation/uploading-documents.mdx
+++ /dev/null
@@ -1,86 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Uploading documents
----
-
-# Uploading documents
-
-
- Learn about how to add files to Aleph, and the ways that Aleph can help to extract meaningful information from documents.
-
-
-The most common type of data to import into an investigation in Aleph is files, such as **PDFs**, **E-Mail archives**, or **Word documents**. Uploading these [documents](/guide/the-basics#documents) to Aleph allows you to keep track of evidence gathered over the course of an investigation and to share it as needed with your colleagues.
-
-Most importantly, uploading files to Aleph makes it easier to search their contents, even if text is hidden in images or other unstructured formats.
-
-## Why upload documents to Aleph?
-
-### Collaboration
-
-Uploading documents to Aleph provides an easy way to **share access** to large sets of documents with your colleagues and collaborators.
-
-### Text Extraction
-
-Aleph **extracts text** from images and PDF files, allowing you to search for key terms of interest in files that would otherwise prove difficult to search.
-
-
-
-### Names, phone numbers, addresses...
-
-Aleph is trained to recognize and extract mentions of **people**, **companies**, **phone numbers**, **addresses**, and **IBAN numbers** contained in documents you upload, allowing you to easily find other documents containing matching mentions.
-
-
-
-### Email archives
-
-When uploading a set of emails, Aleph automatically extracts structured information like the **sender**, **receiver**, **cc'ed entities**, and **attachments**, making it easy to filter the uploaded data for messages sent from or to a specific person.
-
-
-
-### Organized File Structure
-
-Aleph **preserves the folder structure** and hierarchy of uploaded documents, and allows you to create additional folders once files are uploaded into the system. This allows you to keep your files organized as the size of an investigation grows.
-
-
-
-## Getting started
-
-To upload a set of documents, you must first have an [investigation](/guide/building-out-your-investigation/creating-an-investigation) into which to import them. Once you have an investigation ready to go, then proceed with the following:
-
-
-
- From the homepage of your investigation, click the **Documents** button in the sidebar.
-
-
- Then click the **Upload** button to select files or folders from your computer. (Alternatively, **drag** files/folders you would like to upload anywhere onto the screen to start the upload process.)
-
-
-
-
- Decide whether you would like to upload documents with their **folder structure intact**, or as a simple list of files.
-
-
-
-
- Confirm the files you would like to upload, then click the **Upload** button to start the process.
-
-
-
-
-
- It will take a little while to process your uploaded files. In the meantime, you can check the status of the upload by hovering over the name of your investigation at the top of the screen.
-
-
-
-
-
- When the upload is finished, you should now see your files in the **Documents** section of the investigation.
-
-
-
-
-
-## Advanced notes on uploads
-
-- To upload a **large trove of documents** (such as a leak), use the command-line based [alephclient tool](/developers/alephclient) to import documents in an automated fashion.
-- If you plan to import data **on a recurring basis** from a public source, such as a government web site, you may want to [create a web crawler](/developers/memorious) that automatically executes, collects data and submits them to Aleph.
diff --git a/docs/src/pages/guide/building-out-your-investigation/using-the-table-editor.mdx b/docs/src/pages/guide/building-out-your-investigation/using-the-table-editor.mdx
deleted file mode 100644
index 52c8c9fafa..0000000000
--- a/docs/src/pages/guide/building-out-your-investigation/using-the-table-editor.mdx
+++ /dev/null
@@ -1,116 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Creating and editing individual entities
----
-
-# Creating and editing individual entities
-
-
- Aleph provides an easy-to-use spreadsheet interface for creating, editing, linking, and deleting structured entities within an investigation.
-
-
-While entities can be added to an investigation in bulk by [creating network diagrams](/guide/building-out-your-investigation/network-diagrams) or via [generation from an uploaded spreadsheet](/guide/building-out-your-investigation/generating-multiple-entities-from-a-list), it is also possible create and edit entities individually.
-
-To get started, **click one of the entity types** in the sidebar on any investigation page, or select **"Add a new entity type".**
-
-
-
-You will see a table containing all of the entities of that type that have been created within that investigation, **one entity per row**. **Properties** for each entity are displayed in the corresponding columns.
-
-
-
-The table shows only entities of a single type at a time. Use the tabs on the left side to toggle between viewing different types and the **"Add new entity type"** button to add additional types.
-
-## Creating entities
-
-To create a new entity in the Entity Table, either **click the "Add new" button** at the top of the table...
-
-
-
-... or simply **start typing** in once of the cells of the bottom row of the table.
-
-
-
-Then click **"Enter"** to submit the value for the property cell in which you have typed. A new entity will **automatically be created** with the property value you have entered.
-
-## Editing entities
-
-Editing properties of an entity in the table editor works in much the same way as any spreadsheet editor like Microsoft Excel or Google Sheets.
-
-- Use the **arrow keys** or **tab key** to navigate the cells of the table
-- **Click** in a cell or simply **start typing** in an active cell to edit its contents
-- **Click outside** of a cell or press the **enter key** to finish editing a cell
-- **Copy** and **paste** the value of a cell or multiple cells to duplicate values in other cells of the table
-
-### Adding Additional Properties
-
-As mentioned above, each column in the table corresponds to a property of the entities contained within.
-
-By default, the table editor displays the most commonly-used properties of an entity type (i.e. Name, Nationality, and Birth date for a Person), as well as any properties that contain values.
-
-But each entity type has many more additional properties that can be added to the table. Follow these steps to add an additional property:
-
-
- Scroll to the far right of the table.
- Click the **"Add a property"** button.
-
- Then select a property from the list.
-
-
-
-
-
- You will now see the column for your property added to the table. **Click** in a cell to start adding values.
-
-
-
-
-
-### Different property editing types
-
-Properties come in a few different varieties. Most properties, like a person's name or address, are free text values, but other properties, like birth date or nationality are restricted to date or country values, respectively.
-
-When you click to edit a cell, the table editor will display a slightly different editing input, depending upon the type of the property value you are editing. Featured below are a few examples:
-
-
-
-### Adding Multiple Values to a Cell
-
-Most entity properties allow for multiple values, if needed. To add multiple values to a cell, click the **plus (+)** button when editing. Then type an additional value, and press **Enter** to submit.
-
-
-
-## Linking entities
-
-The table editor isn't just useful for creating and editing entities like People, Companies, and Assets. It also provides ways to link those entities together to model the real-world relationships between them.
-
-Follow these steps to add a relationship between two entities:
-
-
- Select an entity to link by **clicking the checkbox** next to it in the table.
-
- Then click the **"Create link"** button.
-
-
-
-
- In the window that appears, select a **target** and a **relationship type** to complete the link. Then click the **"Create"** button.
-
-
-
-
- You'll notice that a new relationship type (in the case below, a Family relationship) has been added to the list of entity types on the left of the table editor. **Click** that relationship type to view the relationship link that you created.
-
-
-
-
-
- You can now add additional details to the link that you created in the same way that you would edit any other entity in the table. This is especially useful for indicating the things like the amount of a shareholding ownership between a Person and a Company entity, or the beginning and end date of a directorship.
-
-
-
-## Deleting entities
-
-To delete one or more entities from the investigation, **click the checkbox** next to each entity you would like to delete. Then click the **"Delete"** button at the top of the table.
-
-
diff --git a/docs/src/pages/guide/frequently-asked-questions.mdx b/docs/src/pages/guide/frequently-asked-questions.mdx
deleted file mode 100644
index c4a346fbfb..0000000000
--- a/docs/src/pages/guide/frequently-asked-questions.mdx
+++ /dev/null
@@ -1,30 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Frequently Asked Questions
----
-
-# Frequently Asked Questions
-
-
- Here we collect common questions asked by users of Aleph. If you are a looking for technical information, also check the [technical FAQ](/developers/technical-faq).
-
-
-## What is a "Legal Entity"?
-
-It's either a company, organisation or person. The term **legal entities** are used when we don't know for sure.
-
-Aleph is often used to search public data, such as companies registries or procurement datasets. Some of these datasets mention directors or suppliers but do not specify whether the given entity is a person, company or even a public body. Since this distinction is sometimes hard to derive from a name alone, we use **legal entity** as a stand-in.
-
-## How can I make a dataset or investigation public?
-
-For quality control purposes, normal users on Aleph cannot make their investigation public. Publishing is reserved for system administrators.
-
-With an admin account \(see [the Technical FAQ on how to set one up](/developers/technical-faq#how-can-i-make-an-admin-user)\), you can open up the dataset settings, and select a different dataset category in that screen. As soon as you select a category other than "Investigation", additional options for provenance become visible. The "Share" dialog will also now list an option to make the dataset public.
-
-## Does Aleph do entity extraction?
-
-Yes. Loads. [It's nerdy](/developers/technical-faq#how-does-aleph-extract-named-entities-from-text).
-
-## Does Aleph have hotkeys / keyboard shortcuts?
-
-Yes. Press the '?' key to see all available hotkeys for the current context.
diff --git a/docs/src/pages/guide/getting-started.mdx b/docs/src/pages/guide/getting-started.mdx
deleted file mode 100644
index 45890403bc..0000000000
--- a/docs/src/pages/guide/getting-started.mdx
+++ /dev/null
@@ -1,34 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Getting Started
----
-
-# Getting started
-
-
- Aleph is a powerful data engine built with investigative reporting in mind. It combines different types of data into a simple search interface.
-
-
-Aleph contains an ever-growing collection of datasets relevant for finding connections among people, companies, and other assets relevant to investigative journalists. Learn how to search through this vast corpus of data by following the steps outlined here:
-
-
-
-With Aleph, organizations can put all kinds of data - from a procurement dataset to a leaked trove of emails - into a single place. This documentation helps people who want to upload, share, and manage data on Aleph.
-
-
-
-Part of the Aleph toolkit has been Aleph Data Desktop, a tool to sketch out networks of companies, people and their relationships on your own computer. As of September 2022, Data Desktop has been deprecated and we do not recommend creating new network diagrams in Data Desktop, but we will keep documentation online for existing users and users looking to export their network diagrams from Data Desktop to Aleph.
-
-
-
-Check out the FAQ to fill in any gaps that are unclear. And feel free to [reach out](/get-in-touch) with any additional questions.
-
-
-
-There's also [a section for developers ](/developers/intro)and people who want to run their own Aleph system.
-
-
-
-If you have problems using Aleph, or want to suggest some future extensions for the toolkit, join our community and chime in:
-
-
diff --git a/docs/src/pages/guide/reconciliation.mdx b/docs/src/pages/guide/reconciliation.mdx
deleted file mode 100644
index 23ee2cb4e3..0000000000
--- a/docs/src/pages/guide/reconciliation.mdx
+++ /dev/null
@@ -1,43 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: OpenRefine Reconciliation
----
-
-# OpenRefine Reconciliation
-
-
- Aleph connects to OpenRefine, a free data processing tool, in order to perform batch lookups for a list of entities (e.g. people and companies).
-
-
-**Entity reconciliation** can be used to check if any entry in a column of data mentions persons or companies of interest.
-
-More generally, the function enables another application - typically [OpenRefine](http://openrefine.org) - to link up (_reconcile_) values in it's own data with the entities stored in Aleph.
-
-To use the reconciliation function, first install the free Open Refine tool on your computer and create a new project from a spreadsheet with at least one column that contains the names of people or companies.
-
-
-
- In the header of that column, select the drop down menu, _Reconcile_, then _Start reconciling..._.
-
-
-
-
- The following window allows you to add a new standard reconciliation service. Paste the URL for Aleph and click _Add Service_.
-
-
- It's recommended to run the reconciliation service against a specific dataset on Aleph, rather than a full site. The reconciliation URL for each dataset is shown on the dataset's overview page.
-
-
-
-
-
- Then choose _Start reconciling_ to begin the reconciliation process.
-
-
-
-
-
-For further information, consult the following documents:
-
-- [Data Augmentation in Refine](https://www.youtube.com/watch?v=5tsyz3ibYzk#t=2m42) (YouTube video).
-- [Reconciliation](https://github.com/OpenRefine/OpenRefine/wiki/Reconciliation) in the Open Refine wiki.
diff --git a/docs/src/pages/guide/search/advanced-search-methods.mdx b/docs/src/pages/guide/search/advanced-search-methods.mdx
deleted file mode 100644
index 398cc6fbad..0000000000
--- a/docs/src/pages/guide/search/advanced-search-methods.mdx
+++ /dev/null
@@ -1,120 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Advanced search
----
-
-# Advanced Search
-
-
- Beyond a simple keyword search, Aleph supports many more complex search operations to find matches based on spelling variations, proximity to other terms, and much more.
-
-
-To view the advanced options available in Aleph, **click the** **advanced search button** next to the search bar. A pop-up will appear explaining each of the available operations.
-
-
-
-Each of the options in the pop-up allows you to enter a term or terms. Once you have entered a term, clicking **Search** executes a search with your updated advanced option. You will notice that each time you enter a new advanced option, the keywords in the main search bar will change, indicating the option you've selected.
-
-We will go through each of the advanced search operations in more detail below.
-
-## Finding an exact phrase or name
-
-By default, Aleph tries to find matches based off your keywords pretty broadly, returning matches that include all of your keywords first, followed by matches that might only include one of your keywords.
-
-For example, if you type the keywords:
-
-```
-Ilham Aliyev
-```
-
-Aleph will return all matches that have the words "Ilham" and "Aliyev," followed by matches that have either "Ilham" or "Aliyev" but not both, in them. Depending on your needs, this might not be ideal.
-
-If you want Aleph to only return matches that have exactly "Ilham Aliyev", then you should put quotations around those two keywords.
-
-```
-"Ilham Aliyev"
-```
-
-
-
-## Allow for variations in spelling
-
-Sometimes a name can be spelled many different ways or even mispelled many different ways. One way to solve this problem is to simply type each variation in the search form:
-
-```
-Aliyev Əliyev Aliyeva Əliyeva
-```
-
-You might capture all the variations you want, but you also might miss some by accident. Another way to tell Aleph to look for variants of a name is to use the \~ operator:
-
-```
-Aliyev~2
-```
-
-What this translates to is: Give me matches that include the keyword Aliyev, but also matches that include up to any 2 letter variations from the keyword Aliyev. These variations include adding, removing, and changing a letter. This includes Aliyev, of course, but also includes Əliyev, which is just one letter variation different, and Əliyeva, which is two letter variations different from Aliyev.
-
-
-
-
- Using this operator with too high a number (greater than 3, for example) will cause slow searches and may return too many false results.
-
-
-## Search for words that should be in proximity to each other
-
-If you do not want to find a precise keyword, but merely specify that two words are supposed to appear close to each other, you might want to use a **proximity search,** which also uses the \~ operator. This will try to find all the requested search keywords within a given distance from each other. For example, to find matches where the keywords Trump and Aliyev are ten or fewer words apart from each other, you can formulate the search as:
-
-```
-"Trump Aliyev"~10
-```
-
-
-
-## Including and excluding combinations of keywords
-
-You can tell Aleph to find matches to multiple keywords in a variety of ways or combinations, otherwise known as a **composite search**.
-
-To tell Aleph that a keyword must exist in all resulting matches, use a + operator. Similarly, to tell Aleph that a keyword must not exist in any of the resulting matches, use - operator.
-
-```
-+Trump -Aliyev
-```
-
-This translates to: Give me all matches in which each match must include the keyword Trump and must definitely not include the keyword Aliyev.
-
-You can take these combinations a step further using the AND operator or the OR operator.
-
-```
-Trump AND Aliyev
-```
-
-This translates to: Give me all matches in which each match must contain both the keywords Trump and Aliyev, but don't return any matches that only contains just one of those keywords.
-
-```
-Trump OR Aliyev
-```
-
-This translates to: Give me all matches in which each match may contain the keywords Trump or Aliyev or both.
-
-You can build on these searches even further like so:
-
-```
-+Aliyev AND (Obama OR Trump) -Georgia
-```
-
-This translates to: Give me all matches in which each match must contain the keyword Aliyev and must contain either the keyword Obama or the keyword Trump, but must not contain the keyword Georgia.
-
-## Putting It All Together
-
-You can combine any of the methods supported by Aleph in many combinations to create some very explicit search rules. The complexity, of course, depends on your needs.
-
-```
-"Ilham Aliyev"~2 AND (Obama OR Trump) -Georgia
-```
-
-## For Dorks
-
-Aleph uses the search engine ElasticSearch in the backend, and many of the operators will work in the search form, but are more advanced. To check out what else is possible, [see ElasticSearch's documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax).
diff --git a/docs/src/pages/guide/search/anatomy-of-a-search.mdx b/docs/src/pages/guide/search/anatomy-of-a-search.mdx
deleted file mode 100644
index 5adcc254d0..0000000000
--- a/docs/src/pages/guide/search/anatomy-of-a-search.mdx
+++ /dev/null
@@ -1,71 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Executing a basic search
----
-
-# Executing a basic search
-
-
- The guide below explains how to carry out a basic search, create email alerts for terms of interest, and how to view and export your results.
-
-
-## The Aleph Search Bar
-
-Aleph's site-wide search bar is located at the top of any page on the site. This search bar allows you to search all of the documents and entities [**you have access to**](/guide/the-basics#groups) in Aleph. It is comprised of several distinct features:
-
-
-
-### The Search Form
-
-The **search form** is where you enter keywords and other terms of interest in order to execute your search.
-
-### Create Alert Button
-
-The **create alert button** enables you to create a saved alert based on the keywords and operators in the search form. Once saved, you will receive email notifications as well as see alerts in the notification tab when new data is added to Aleph that meet the requirements of your search.
-
-### Advanced Search Button
-
-The **advanced search button** displays a popup form which allows you to apply various [**advanced operators and methods**](/guide/search/advanced-search-methods) to your search, like fuzzy matching, spelling variations, and proximity searches.
-
-## Viewing search results
-
-When you execute a search, the results of your search are listed below the search bar. As you scroll, more of the results will be displayed, until you reach the end of the list. The **result count** lets you know how many results in total were found matching your term.
-
-
-
-With each result, **yellow highlighted text** indicates the context in which your search term was found. This allows you to eyeball the results and understand why they were returned based on your keywords. This can also help you decide if you need to either expand or narrow down your search.
-
-## Previewing a result
-
-**Clicking on a search result** opens a preview of the document or entity in question, allowing you to briefly assess whether it is worth a more in-depth look. You can quickly preview many results by using the **up and down arrow keys** to scroll between entries.
-
-
-
-In the preview, clicking the **Expand** button will take you to the main page for that entity or document, while the **Download** button (for documents and other downloadable content) allows you to save a copy to your local machine. The **X** button closes the preview, returning you to the main results view.
-
-## Filtering results
-
-Aleph's search results page includes a list of filters to further narrow down your search results. These filters are located in the left-hand sidebar of the page. Learn more about filtering your results [**here**](/guide/search/filtering-your-search-results)**.**
-
-
-
-## Exporting your results
-
-Aleph enables you to download the results of your search as a spreadsheet. Just **click on the export button** in the top-right of the window to download the results. Aleph limits the number of search results you can download to a **maximum of 10,000 results**.
-
-
diff --git a/docs/src/pages/guide/search/filtering-your-search-results.mdx b/docs/src/pages/guide/search/filtering-your-search-results.mdx
deleted file mode 100644
index 2da9659c55..0000000000
--- a/docs/src/pages/guide/search/filtering-your-search-results.mdx
+++ /dev/null
@@ -1,38 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Filtering your search results
----
-
-# Filtering your search results
-
-
- With so much data to sift through, Aleph gives you many options for filtering your search to only the content types, datasets, countries, and so on that you are most interested in.
-
-
-Once you have honed your search using the [**advanced search methods**](/guide/search/advanced-search-methods) described earlier, you can further filter the search results by a number of built in filters.
-
-Aleph provides built-in options to filter your search results by **country**, **type of data** your are interested in (document, company entity, web page, image, video, etc.), **filetype**, **specific datasets**, **email addresses**, **phone numbers**, **bank accounts**, **names**, **language**, and more.
-
-
-
-Each filter type provides a list of checkboxes that allow you to select exactly what you most want to see. In addition, the checkbox options are **sorted from most prevalent to least** with the quantity of occurrences beside it. This number can be quite useful, as it shows the most common options first, giving you an indicator of where to start filtering to narrow down your search.
-
-When you **select a filter value by clicking** in the checkbox, a button appears above the search results indicating that the results are being filtered by that option.
-
-If you want to remove the filter, it is easy enough to **click on the "X" in the button to remove it** or click on the "Clear All" button to remove all of the filters at once.
-
-
-
-While most filters allow you to select multiple options at once to view options that fit either one or both of the selected filter criteria, **the "Types" filter currently only supports selecting a single content type** at a time.
-
-## Filtering By Date
-
-Clicking the **Date distribution** button allows you to browse a chronological view of your results. This visual distribution can be useful to get a sense of the spread or _shape_ of your data.
-
-The displayed chart organizes your results according to any date type properties that a given result may have. This could include, for example, a Person's birth date, or a Company's incorporation date.
-
-To filter your results by year, simply **click one of the bars** in the chart or **click and drag across** the bar chart to select a range of years.
-
-
-
-Just like the other filters, when you filter the data by a date range, filter buttons appear at the top of the bar chart, enabling you to see what filters are currently being applied and remove them if you no longer need them.
diff --git a/docs/src/pages/guide/search/index.mdx b/docs/src/pages/guide/search/index.mdx
deleted file mode 100644
index c2cdbcbb05..0000000000
--- a/docs/src/pages/guide/search/index.mdx
+++ /dev/null
@@ -1,36 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Searching your data
----
-
-# Searching your data
-
-
- Aleph can do more than just keyword search. This guide explains some of the more advanced queries supported by the system, as well as how to filter and refine your results.
-
-
-Aleph stores vast numbers of [**documents**](https://docs.alephdata.org/guide/the-basics#documents), [**entities**](https://docs.alephdata.org/guide/the-basics#entities), and the relationships between entities across many [**datasets**](https://docs.alephdata.org/guide/the-basics#datasets). Here we'll explore how to narrow down or expand your search to methodically find the answers to your questions or just stumble on an interesting lead using Aleph's advanced set of search operators.
-
-
-
-Get started by **learning the basics** of searching for documents and entities across the entire corpus of data you have access to in Aleph.
-
-
-
-Learn about advanced methods to restrict your results to only **exact matches**, or to broaden your results to include **spelling variations** or **fuzzy matches**.
-
-
-
-Looking for only **a** **specific type of content**, like PDF documents or companies? Or **within a fixed date range**? Or only for matches that **contain a specific address**? Learn how to apply filters to further hone your results.
-
-
-
-Finally, learn about how to search **in more contained contexts**, like within the contents of an individual dataset or document.
-
-
-
-## Searching for Datasets
-
-The methods above are all focused on searches for **entities** and **documents** within Aleph, but what about searching for **a specific dataset** or for all of the datasets pertaining to a country?
-
-
diff --git a/docs/src/pages/guide/search/searching-for-a-dataset.mdx b/docs/src/pages/guide/search/searching-for-a-dataset.mdx
deleted file mode 100644
index f9cc3fe797..0000000000
--- a/docs/src/pages/guide/search/searching-for-a-dataset.mdx
+++ /dev/null
@@ -1,32 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Searching for a dataset
----
-
-# Searching for a dataset
-
-
- All of the previous guides have focused on searching for documents or entities in a variety of contexts in Aleph. But what if you'd like to know what datasets are even available?
-
-
-The main search bar at the top of the Aleph site returns results for documents and entities, but **not datasets**. In order to search for a specific dataset, say, the UK Companies House or the Panama Papers, you must click the **Datasets button**.
-
-
-
-You will then be brought to a list of all of the [**source datasets**](/guide/the-basics#source-datasets) to which you have access in Aleph. At the top of this screen you will see a **search bar** which allows you to enter keywords to find specific datasets, as well as a **result count**, and buttons to **sort the results.**
-
-
-
-Each dataset listing includes a **brief description**, a note of **when it was last updated**, the **countries** it pertains to, the count of **how many document and entities** it contains, and, in some cases, the **frequency it is updated** and **how restricted its access is**.
-
-Clicking a dataset listing will bring you to that dataset's homepage, allowing you to view more detailed metadata about the dataset, an overview of its contents and the ability to search within and cross-reference with other datasets.
-
-## Filtering the datasets list
-
-In much the same way as the [**filters**](/guide/search/filtering-your-search-results) used when searching for entities and documents, the datasets list can be filtered either by **Category** or **Country**.
-
-
-
-In this way you can search for all of the datasets pertaining to Panama, or all of the Company registers for a given country.
-
-**Now that you've learned all there is to know about searching pre-existing source data in Aleph, read further to learn about creating and uploading your own!**
diff --git a/docs/src/pages/guide/search/searching-within-other-contexts.mdx b/docs/src/pages/guide/search/searching-within-other-contexts.mdx
deleted file mode 100644
index bfa5ed3043..0000000000
--- a/docs/src/pages/guide/search/searching-within-other-contexts.mdx
+++ /dev/null
@@ -1,46 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: Searching within other contexts
----
-
-# Searching within other contexts
-
-
- In addition to the Aleph site-wide search which we have covered, there are a handful of additional, more precise contexts in which you can search.
-
-
-## Searching within a single dataset
-
-On the homepage for a source dataset you will notice a search bar in the top right corner. This search bar allows you to search the contents of that dataset.
-
-
-
-Additionally, various overview statistics highlight **the most frequently appearing names**, **addresses**, **types of entities**, and more. Clicking one of these values will automatically bring you matches for that specific value within the dataset.
-
-The results returned for a intra-dataset search look much the same as those returned for the Aleph site-wide search, and can be [**filtered**](/guide/search/filtering-your-search-results) and refined in exactly the same way.
-
-
-
-## Searching within a folder
-
-Datasets or investigations that are organized hierarchically allow you to search just within the contents of a given folder or package within the dataset.
-
-
-
-## Searching within a document
-
-One of the most valuable assets that Aleph provides is the ability to extract text from less search-friendly document types like PDFs. Viewing these documents in Aleph allows you to search within the extracted text contents of the original.
-
-
-
-Results for such searches are returned by the page on which they were found.
-
-
-
-## Searching within a network diagram
-
-It is possible to search the contents of a network diagram by entering a search term in the diagram search bar in the upper right corner above the diagram.
-
-
-
-**In this way, Aleph provides multiple different search contexts to allow you to focus your search within a dataset, folder, document, or diagram of specific interest.**
diff --git a/docs/src/pages/guide/the-basics.mdx b/docs/src/pages/guide/the-basics.mdx
deleted file mode 100644
index f9d94e1ad1..0000000000
--- a/docs/src/pages/guide/the-basics.mdx
+++ /dev/null
@@ -1,60 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: The building blocks of Aleph
----
-
-# The building blocks of Aleph
-
-
- This page defines some useful terms for understanding the basics of Aleph and how the Aleph ecosystem is structured to support your investigations.
-
-
-Aleph is more than just a place to search and explore **source documents**. It also provides **structured entities** to model the primary units of an investigation, **datasets** to house both source material and investigation files, and **groups** to help you share and manage access to knowledge as your investigation grows.
-
-## Documents
-
-With Aleph, you can [**search**](/guide/search), [**upload**](/guide/building-out-your-investigation/uploading-documents)**, organize**, and **share** many different types of documents that would otherwise be difficult to explore. Aleph provides support for extracting meaningful information from PDFs, Excel files, CSVs, email archives, web sites, and other document formats to make source material easier to browse and analyze no matter in which form it comes.
-
-While documents often provide the evidentiary backbone of an investigation, Aleph is built to explore complicated relationships that are often difficult to glean from documents alone - this is where structured entities come in.
-
-## Entities
-
-Aleph provides an ever-growing [**vocabulary**](/developers/followthemoney) for describing and modeling the **people**, **companies**, **assets**, and **relationships** that are often at the center of your investigations.
-
-
-
-Each entity type contains a fixed set of possible properties to describe relevant details of that entity, and can be linked to other entities via relationships.
-
-
-
-This structured vocabulary allows entities to be more easily searched, filtered, and cross-referenced with other data sources to find relevant co-occurrences and further enrich your investigation.
-
-## Datasets & Investigations
-
-In Aleph, **datasets** and **investigations** serve as the primary containers for organizing and managing collections of documents and entities. Any document or entity in Aleph must be a part of either a dataset or an investigation.
-
-### Source datasets
-
-**Source datasets** reflect the accumulated contents of a single source (i.e. a company registry, an email leak, a court archive). They are managed by the data administrators of Aleph and cannot be changed or edited by any other users of the platform. Learn about how to find a specific dataset or datasets pertaining to a specific country [**here**](/guide/search/searching-for-a-dataset).
-
-
-
-### Investigations
-
-**Investigations** are contained workspaces within Aleph where you can upload, edit, and organize data related to an investigation or topic of interest - and they can be shared with any other user or access group within Aleph.
-
-While source datasets cannot be modified, **investigations** allow you to edit their contents by [**uploading documents**](/guide/building-out-your-investigation/uploading-documents), [**mapping data into structured entities**](/guide/building-out-your-investigation/generating-multiple-entities-from-a-list), and [**creating your own network diagrams**](/guide/building-out-your-investigation/network-diagrams).
-
-Read more about how to get started creating and managing an investigation [**here**](/guide/building-out-your-investigation/creating-an-investigation).
-
-## Groups
-
-While browsing Aleph, you are only able to view or edit data according to the respective access permissions you have been granted.
-
-Access in Aleph is managed through **Groups**. Once granted permission, members of a group have access to all of the **source** **datasets** and **investigations** that have been shared with the group.
-
-Groups are managed by site administrators, and cannot, therefore, be created or deleted through the Aleph user interface.
-
-Read more about managing access [**here**](/guide/building-out-your-investigation/creating-an-investigation#managing-access-to-your-investigation).
-
-**Now that you've gotten acquainted with some of the core concepts of Aleph, it's time to dive in. Read on to learn more about** [**searching in Aleph**](/guide/search) **and** [**building out your own investigation**](/guide/building-out-your-investigation)**.**
diff --git a/docs/src/pages/how-aleph-is-used.mdx b/docs/src/pages/how-aleph-is-used.mdx
deleted file mode 100644
index 4ecb39354c..0000000000
--- a/docs/src/pages/how-aleph-is-used.mdx
+++ /dev/null
@@ -1,57 +0,0 @@
----
-layout: '@layouts/DocsLayout.astro'
-title: About the project
----
-
-# About the project
-
-
- Aleph is developed mainly for use by investigative reporters in cross-border investigations.
-
-
-The goal of the Aleph project is to provide powerful technology to investigative journalists who want to track \(mainly\) white collar crime. The development is fully open, and coordinated by the data team at the [Organized Crime and Corruption Reporting Project](https://occrp.org). We welcome contributors from other contexts, such as anti-corruption advocacy.
-
-## Why was Aleph created?
-
-We decided to build Aleph to solve the following principal needs:
-
-- Easy **data search for both structured and unstructured information** \(ie. documents and databases\).
-- **Cross-referencing** between different datasets \("Who are all the politicians in my country that are mentioned in this leak?"\)
-- **Access control** and data compartmentalisation, but also flexible sharing within cross-border teams.
-- Visual exploration of investigative analysis through network diagrams, timelines and other investigative tools.
-- Continuous crawling of hundreds of public data sources as background material for research.
-
-Our goal with all this has been to make an open, modular and extensible toolkit that can be operated at scale and in production.
-
-## Who is using Aleph?
-
-Most of the organisations that have adopted Aleph do so for in-house use. Some public instances exist:
-
-- Organized Crime and Corruption Reporting Project[ \(OCCRP\) Aleph](https://aleph.occrp.org)
-- Balkans Investigative Reporting Network \(BIRN\) - [Research Desk](https://source.bird.tools/)
-- DDoSecrets [Hunter Memorial Library](https://hunter.ddosecrets.com/)
-- Code for Africa [gazeti.africa](https://gazeti.africa/)
-
-Organisations with in-house Alephs include Global Witness, Süddeutsche Zeitung \(SZ\), Swedish Radio \(SR\), Norddeutscher Rundfunk \(NDR\) and others.
-
-## How can I contribute?
-
-Aleph is an open source project, we're very excited for contributions from people or organisations who see benefit in helping to refine or extend our tools. Here are some of the ways you could help:
-
-- If you are a **native speaker of a language** that isn't supported by the Aleph user interface, join the [Transifex project](https://www.transifex.com/aleph/) and help us provide the software in that language.
-- If you're a **coder** \(Python, JavaScript/React.js\), jump on our issue tracker to see open issues. Work that could be a good entry point for a new contributor is marked [**help-wanted**](https://github.com/alephdata/aleph/labels/help-wanted).
-- If you're an **open data enthusiast or data scraper**, you can convert data from a public source to our [FollowTheMoney data format](/developers/followthemoney/) and share the result with [the community](/get-in-touch). Consider contributing a [memorious](/developers/memorious) scraper to the Aleph [data commons](/developers/datacommons)!
-- Any **graphic designers or visual artists** could help us immensely, in particular by improving the set of icons available inside the application. Please [get in touch](/get-in-touch)!
-
-## Who has funded the development?
-
-Aleph is the product of a collaborative effort, and many on the team been able to invest time and effort into developing the product. The following organisations have made this possible:
-
-- The [Google Digital News Initiative](https://newsinitiative.withgoogle.com/) has supported OCCRP with a [large-scale grant](https://newsinitiative.withgoogle.com/dnifund/dni-projects/turnkey-data-platform-investigative-teams/) to build Aleph. Google Ideas \(now Jigsaw, an Alphabet unit\) also provided funding for the Investigative Dashboard, some of which helped to kickstart this project.
-- Development of Aleph is also financed via OCCRP's core grants from the US Agency for International Development \(USAID\), the US Department of State \(DRL\), Open Societies Foundations \(OSF\), and Luminate.
-- _If you would like to support OCCRP's development of Aleph, please contact our Partnerships team via info@occrp.org._
-
-Besides grants to OCCRP, the following other organisations have contributed to the development of Aleph:
-
-- The initial prototyping of Aleph was supported by the International Center for Journalists \(ICFJ\) through a [Knight International Journalism Fellowship](https://www.icfj.org/our-work/knight/profiles/friedrich-lindenberg) in 2014/2015.
-- EU Project ODINE supported the use of [Aleph by OpenOil UG](https://opendataincubator.eu/category/openoil/) via its data startup incubator programme.
diff --git a/docs/src/pages/index.astro b/docs/src/pages/index.astro
new file mode 100644
index 0000000000..99a1b41949
--- /dev/null
+++ b/docs/src/pages/index.astro
@@ -0,0 +1,58 @@
+---
+import {
+ Skeleton,
+ Wrapper,
+ Youtube,
+ Wrapper,
+} from 'astro-theme-docs/components';
+import options from '../options.json';
+import HomeTeasers from '@components/HomeTeasers.astro';
+import HomeAbout from '@components/HomeAbout.astro';
+import HomeFeatures from '@components/HomeFeatures.astro';
+import HomeContact from '@components/HomeContact.astro';
+---
+
+
+
+
+
+
+
+ Welcome to the Aleph User Guide and Technical Documentation
+
- Welcome to the documentation for Aleph, a suite of data analysis tools for investigators.
-
-
-Aleph is a powerful tool for people who follow the money. It helps investigators to **securely access and search large amounts of data** - no matter whether they are a government database or a leaked email archive.
-
-
-
-## Aleph for users
-
-If you're a reporter or other user of Aleph, you can find out **how to search Aleph**, **sketch out networks of people and assets**, **upload your own data**, or **link Aleph with other tools**.
-
-
-
-
-
-## Aleph for developers and admins
-
-From a technical perspective, Aleph is a whole **suite of open source tools**. Besides running and configuring the Aleph server, the documentation also covers separate tools that are used to process data flexibly, interact with Aleph, or export data to another format.
-
-
-
-## About the name
-
-Aleph is named after a short story written by the Argentinian author Jorge Luis Borges, [The Aleph](http://www.phinnweb.org/links/literature/borges/aleph.html). It tells of a magical place that encapsulates all other places:
-
-_Truth cannot penetrate a closed mind. If all places in the universe are in the Aleph, then all stars, all lamps, all sources of light are in it, too._
diff --git a/docs/src/pages/users/advanced/reconciliation/index.mdx b/docs/src/pages/users/advanced/reconciliation/index.mdx
new file mode 100644
index 0000000000..a393a89943
--- /dev/null
+++ b/docs/src/pages/users/advanced/reconciliation/index.mdx
@@ -0,0 +1,40 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: OpenRefine Reconciliation
+---
+
+# OpenRefine Reconciliation
+
+
Aleph connects to OpenRefine, a free data processing tool, in order to perform batch lookups for a list of entities (e.g. people and companies).
+
+Entity reconciliation can be used to check if any entry in a column of data mentions persons or companies of interest. More generally, the function enables another application - typically [OpenRefine](http://openrefine.org) - to link up (reconcile) values in it's own data with the entities stored in Aleph.
+
+To use the reconciliation function, first install the free Open Refine tool on your computer and create a new project from a spreadsheet with at least one column that contains the names of people or companies.
+
+
+
+ In the header of that column, select the drop down menu, **Reconcile**, then **Start reconciling…**.
+
+
+
+
+ The following window allows you to add a new standard reconciliation service. Paste the URL for Aleph and click **Add Service**.
+
+
+ It's recommended to run the reconciliation service against a specific dataset on Aleph, rather than a full site. The reconciliation URL for each dataset is shown on the dataset's overview page.
+
+
+
+
+
+ Then choose **Start reconciling** to begin the reconciliation process.
+
+
+
+
+
+For further information, consult the following documents:
+
+* [Data Augmentation in Refine](https://www.youtube.com/watch?v=5tsyz3ibYzk#t=2m42) (YouTube video).
+* [Reconciliation](https://github.com/OpenRefine/OpenRefine/wiki/Reconciliation) in the Open Refine wiki.
+
diff --git a/docs/src/pages/users/faq/account/index.mdx b/docs/src/pages/users/faq/account/index.mdx
new file mode 100644
index 0000000000..7f73af48b3
--- /dev/null
+++ b/docs/src/pages/users/faq/account/index.mdx
@@ -0,0 +1,76 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: Account & Recovery
+---
+
+# Account creation and password recovery
+
+## When I enter the code from my 2FA app to log in to Aleph, it says it is incorrect. I tried several times and it did not work. What should I do?
+
+If you are receiving an error message after putting in your one time password, this issue is most likely caused by the current time on your phone or laptop. Follow the troubleshooting steps below:
+
+### Using macOS?
+
+
+
+ Click on the Apple logo in the upper left-hand corner and select **System preferences** from the dropdown menu. In the System Preferences, go to **Date and Time**.
+
+
+
+ Click the **lock icon** in the bottom left-hand corner of the window, then enter your administrator password to unlock the lock. (The settings are dimmed when locked).
+
+
+
+ In the **Date and Time** pane, make sure that **Set date and time automatically** is selected and your Mac is connected to the internet. Your Mac can then get the current date and time from the network time server selected in the adjacent menu.
+
+
+
+### Using Windows?
+
+
+
+ To set your time and time zone in Windows 10, go to **Start** ▶ **Settings** ▶ **Time & language** ▶ **Date & time**.
+
+
+
+ Turn on **Set time automatically** and **Set time zone automatically**.
+
+
+
+### Using an iPhone?
+
+
+
+ Go to **Settings** ▶ **General** ▶ **Date and Time**.
+
+
+
+ Turn on **Set Automatically**. The iPhone now gets the correct time over the network and updates it for the timezone you are in. Some networks don't support network time, so in some countries or regions, an iPhone may not be able to automatically determine the local time.
+
+
+
+### Using Android?
+
+
+
+ Go to **Settings** and click on **Date and Time**.
+
+
+
+ Click on **Automatic**. If this option is turned off, check that the correct **Date**, **Time**, and **Time zone** are selected.
+
+
+
+If you followed the above troubleshooting steps and are still experiencing issues, please contact OCCRP’s IT Helpdesk using [this form](https://form.asana.com/?k=9e-nT6JyUfSivzqHwNhsDw&d=24418422500834).
+
+## I already have an Aleph account but I do not remember my username. Is there a way to recover it?
+
+Your Aleph account’s username is the e-mail address you used to register. If you are entering the correct e-mail address and are experiencing issues logging into Aleph, please contact OCCRP’s IT Helpdesk using [this form](https://form.asana.com/?k=9e-nT6JyUfSivzqHwNhsDw&d=24418422500834).
+
+## I removed my 2FA app/lost my phone and I cannot log in to Aleph. What should I do?
+
+In either of the above situations, you will have to reset the password to your Aleph account. Please follow the instructions to [reset your password](http://localhost:3000/users/getting-started/account#reset-your-password).
+
+## I tried everything and I can’t log in to Aleph? How do I get in touch with OCCRP to ask for assistance?
+
+If you followed all of the above troubleshooting steps and are still experiencing issues accessing your Aleph account, please contact OCCRP’s IT Helpdesk using [this form](https://form.asana.com/?k=9e-nT6JyUfSivzqHwNhsDw&d=24418422500834).
diff --git a/docs/src/pages/users/getting-started/account/index.mdx b/docs/src/pages/users/getting-started/account/index.mdx
new file mode 100644
index 0000000000..3ad152f485
--- /dev/null
+++ b/docs/src/pages/users/getting-started/account/index.mdx
@@ -0,0 +1,119 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: Manage your account
+---
+
+{/*TODO Use page/screen consistently*/}
+
+# How to create an account or reset your password
+
+
On this page, you will find instructions on how to create an account for OCCRP Aleph and how to reset your password.
+
+{/*TODO Add warning: Sign up / sign in may be different for in-house instances*/}
+
+
+
+## Create an account
+
+
+ To create an Aleph account, go to [aleph.occrp.org](https://aleph.occrp.org/) and click on Sign In in the upper right-hand corner.
+
+
+
+
+
+
+
+
+
+ You will be redirected to the **OCCRP Secure Sign-in** page. Once there, click on New user? Register.
+
+
+
+
+
+
+
+
+ Aleph will ask you for an e-mail address and a password, among other information. Subsequently, you will receive an e-mail confirming that your account has been created. Click on the link received to complete the e-mail verification process.
+
+
+
+ Now, try logging in to your new account with the e-mail and password you chose. For security reasons, Aleph requires you to have an authentication application installed, which works as a two-step verification to log in to the account. We recommend using the free **Google Authenticator** application, which can be downloaded directly from your device’s app store.
+
+
+
+ Within the app, click on the **+** symbol followed by **Scan a QR code** and scan the QR code displayed on your screen.
+
+
+
+
+
+
+
+
+ Finally, enter the temporary code displayed on your mobile phone. For example:
+ {/*TODO Clarify what to enter in which fields*/}
+
+
+
+
+
+
+ Having problems creating an Aleph account? Please, check our [FAQ section](/users/faq/account).
+
+
+## Reset your password
+
+
+
+ Go to [aleph.occrp.org](https://aleph.occrp.org) and click on **Sign In** in the upper right-hand corner.
+
+
+
+ You will then be presented with the following window. In the bottom right-hand corner, go to Forgot Password?
+
+
+
+
+
+
+
+
+ On a new screen, Aleph will ask you to enter your e-mail address (the e-mail address you used to register on the platform).
+
+
+
+
+
+ You will then receive an e-mail titled **Link to reset credentials**. When you click on it, a new screen will open asking you to scan a code with the app you used when you created your user (e.g. **Google Authenticator**). Then enter the code provided by the app (**One-time code** field) and finally define a name for the phone you are using (**Device name** field). You can put any name (please remember it), that's all!
+
+
+
+
+ Having problems recovering your username or password? Please, check our [FAQ section](/users/faq/account). If you do not see the Aleph email in your inbox, please remember to check your spam folder.
+
+
+## Request extended data access
+
+Are you a journalist, a researcher or an activist? We have created special user groups to support reporters, journalists, researchers and NGOs respectively. These groups contain hundreds of datasets for you to work with. You can apply for this extended access [using this form](https://form.asana.com/?k=hsYmAKHX1ViTzUoe410y8Q&d=24418422500834). Our Research & Data team evaluates applications case by case and responses will be handled in a timely manner.
diff --git a/docs/src/pages/users/getting-started/homepage/index.mdx b/docs/src/pages/users/getting-started/homepage/index.mdx
new file mode 100644
index 0000000000..6adde9e8b1
--- /dev/null
+++ b/docs/src/pages/users/getting-started/homepage/index.mdx
@@ -0,0 +1,161 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: Navigate the homepage
+---
+
+{/*TODO Use page/screen consistently*/}
+
+# How to navigate the Aleph homepage
+
Welcome to the Aleph homepage! The homepage appears once you log in to your Aleph account. It is divided into separate sections, each with its own functionality. On this page, you will learn more more about the different sections.
+
+
+
+## Top menu
+
+
+
+
+
+
+
+
+
+
+
+### Search box
+This is the general search box. All terms that are typed in the search box will result in Aleph searching for information across all datasets which you have access to. The gray symbol to the right of the search box represents the [advanced search feature](/users/search/advanced).
+
+### Datasets
+By clicking on this button, you can view all the datasets accessible for your account.
+
+### Investigations
+In this section, you can create your own investigation workspace to upload documents, cross-reference lists of names and persons and build relationship diagrams, among other functionalities. By default, these are private workspaces, i.e., associated with your account. However, you can share your research with other users who have an Aleph account.
+
+### Bookmarks
+In this section, you can view which documents you bookmarked throughout your search.
+
+### About
+In this section, you are able to find out more about the Aleph project, our acceptable use terms, frequently asked questions, and terms of use.
+
+### Account settings
+By clicking your profile name in the upper right-hand corner, you can view notifications, alerts, exports, investigations, network diagrams, timelines, lists, settings, system status, and the option to sign out. See side menu explanation below for more information.
+
+## Side menu
+Similar to the account settings option available in the upper right-hand corner of your Aleph screen, the left-hand side menu provides similar functionality.
+
+### Activity
+
+
+
+
+
+
+#### Notifications
+By default, when on the homepage, you will be able to see updates made to the datasets available to you.
+
+#### Alerts
+You can save the names of people, e-mails, and/or companies and ask Aleph to notify you if the term is mentioned in new datasets that are uploaded or in updated datasets. These alerts are private to you and are only accessible to the profile that generates them. See the Creating alerts section of this user guide for more information on alerts.
+
+#### Exports
+The results of the searches that you perform in Aleph can be downloaded. This button will display the exports generated by you.
+
+### Workspace
+
+
+
+
+
+
+#### Investigations
+Summarizes the number of investigation workspaces created by you. See [Creating your investigation workspace](/users/investigations/create) for more information on investigations.
+
+#### Network diagrams
+Network diagrams let you illustrate webs of people, companies, bank accounts and other objects in a visual way. See [Drawing network diagrams](/users/investigations/network-diagrams) for more information on network diagrams.
+
+#### Timelines
+Timelines allow you to visualize events by date.
+
+{/* TODO Link to timelines documentation article */}
+{/* See [Creating timelines](/users/investigations/timelines) for more information on timelines. */}
+
+#### Lists
+{/*TODO Check if we should link to the xref guide*/}
+Shows the current number of lists that you have created. Lists let you organize and group related entities of interest. See [Uploading lists to cross-reference your data against other datasets](/users/investigations/cross-referencing) for more information on lists.
+
+### Groups
+
+
+
+
+
+
+This section shows which research projects/groups OCCRP granted access for you to view. For further details, see [Request privileged data access](/users/getting-started/account#request-extended-data-access).
diff --git a/docs/src/pages/users/getting-started/key-terms/index.mdx b/docs/src/pages/users/getting-started/key-terms/index.mdx
new file mode 100644
index 0000000000..fad3b0515d
--- /dev/null
+++ b/docs/src/pages/users/getting-started/key-terms/index.mdx
@@ -0,0 +1,94 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: Key terms
+---
+
+import Listing from '@components/Listing.astro';
+import SchemaListing from '@components/SchemaListing.astro';
+
+# Key terms you need to know before using Aleph
+
+
This page defines some useful terms for understanding the basics of Aleph.
+{/*TODO Update lead*/}
+
+## FollowTheMoney
+
+Underneath Aleph is FollowTheMoney, a data model that helps us organize all datasets we upload in a similar way so that Aleph can provide you with useful leads and patterns for your investigations. Understanding some of the key elements of the FollowTheMoney model can be useful to get the most out of Aleph.
+
+### Entity
+An entity in Aleph can be a person, a company, an asset, a vessel, a contract, an event, a bank account, etc. You can find the full list of entity types available in Aleph below:
+
+
+
+An entity also includes the relationships between objects, such as a payment between two people, or the ownership of a company by a person or another company. The following relationship types are available in Aleph:
+
+
+
+### Properties
+Both entities and relationships can have properties. For example, a person can have a **full name**, **date of birth**, and **ID number**. The ownership of a company can have a start and end date.
+
+What do we mean by this? Let’s take a look at a real example. In the [UK People with significant control](https://aleph.occrp.org/datasets/2053) dataset in Aleph we can find a person (entity) called [Mr. Donald John Trump](https://aleph.occrp.org/entities/f5d2d346fa6f47f4c93d7bbbe423cc6b4b15c68b.6254fb2e28841278da8fdac6bccfe7bd89265c91), former president of the United States. For this entity, Aleph offers more details to users. We can observe that Trump’s nationality and country of origin is the **United States**, his birth date is **1946-06** and the address linked to him is **725, Fifth Avenue, New York**:
+
+
+
+All the listed elements above are properties or characteristics associated with Donald John Trump.
+
+### Legal Entity
+
+It's either a company, organization or person. The term **legal entities** are used when we don't know for sure. Some of the datasets in Aleph mention directors or suppliers but do not specify whether the given entity is a person, company or even a public body. Since this distinction is sometimes hard to derive from a name alone, we use the term legal entity as a stand-in.
+
+## Dataset
+
+A dataset in Aleph is a collection of documents or entities. For instance: going back to our previous example, that would be the [UK People with significant control](https://aleph.occrp.org/datasets/2053) database, since it reunites a list of people (entities).
+
+Aleph compiles a diverse range of databases. Each of the datasets uploaded to Aleph is associated with a country/countries and a category that summarizes the types of data it corresponds to.
+
+This is the list of all categories of datasets in Aleph:
+
+
+
News archives
+
Leaks
+
Land registry
+
Gazettes
+
Court archives
+
Company registries
+
Sanctions lists
+
Procurement
+
Financial records
+
Grey literature
+
Document libraries
+
Licenses and concessions
+
Regulatory filings
+
Persons of interest
+
Customs declarations
+
Population census
+
Air and maritime registers
+
Other material
+
+
+
+
+## Cross-referencing
+
+Since datasets in Aleph are organized following a particular data model, entities from one dataset can be compared against entities of other datasets that you have access to. This process is called cross-referencing and allows journalists to find leads and patterns across hundreds of datasets.
+
+
+
+## Investigations
+
+Investigations are workspaces where you can upload, edit, and organize data (entities or documents) related to a project or topic of interest. They contain datasets with additional features. They can be shared with any other user or [access group](#groups) within Aleph. You can upload documents, create entities, network diagrams, timelines and cross-reference a particular list of people or companies against other datasets in Aleph.
+
+
+
+## Groups
+
+Access in Aleph is managed through groups. Once granted permission, members of a group have access to all of the source datasets and investigations that have been shared with the group. Groups are managed by Aleph administrators who are responsible for their creation and deletion.
diff --git a/docs/src/pages/users/index.mdx b/docs/src/pages/users/index.mdx
new file mode 100644
index 0000000000..bd2208151d
--- /dev/null
+++ b/docs/src/pages/users/index.mdx
@@ -0,0 +1,35 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: User Guide
+---
+
+# User Guide
+
+
Welcome to the Aleph User Guide. The user guide helps you getting started with Aleph and has step-by-step instructions that help you get the most out of Aleph.
+
+## Getting started
+
+New to Aleph? Make sure to read the following pages to get started:
+
+
+
+
+
+## Search
+
+Learn how to use Aleph’s powerful search features to search through hundreds of datasets.
+
+
+
+
+
+## Investigations
+
+Find out how to work with investigation workspaces to uncover insights in your own data and documents and collaborate with colleagues.
+
+
+
+
+
+
+
diff --git a/docs/src/pages/users/investigations/create/index.mdx b/docs/src/pages/users/investigations/create/index.mdx
new file mode 100644
index 0000000000..bcc3bd2558
--- /dev/null
+++ b/docs/src/pages/users/investigations/create/index.mdx
@@ -0,0 +1,62 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: Creating an investigation workspace
+---
+
+# Creating your investigation workspace
+
+
This page explains how your can create your own investigation workspace and (optionally) share it with colleagues and other collaborators.
+
+
+
+ There are two ways to create your own investigation workspace in Aleph. You can either click on the **Investigations** button in the top menu bar:
+
+
+
+ Or you can use the **Investigations** button located on the left sidebar of the homepage.
+
+
+
+
+
+ Once you click on **New Investigation**, the following window will pop-up:
+
+
+
+
+
+ Please fill in the required fields. That is, assign a name to the project or research, add a description and specify the languages of the documents in the **Languages** field. Specifying a language helps Aleph extract text from uploaded documents with higher accuracy, especially if the documents contain text in non-Latin characters.
+
+ When you create an investigation workspace, its contents are by default accessible to you and no one else. However, In the **Share with** field you can enter the Aleph user e-mails of the colleagues you want to share the investigation with.
+
+
+
+ Click **Save**.
+
+
+
+ After that, you will be redirected to your new investigation workspace:
+
+
+
+ The options in the center of the screen tell us what we can do in our own investigation workspace:
+
+ * Upload documents
+ * Create new entities (names of people or companies that we want to cross-reference against Aleph datasets to find matches)
+ * Create network diagrams
+ * Compare our lists against the datasets to which our profile has access
+
+
+
diff --git a/docs/src/pages/users/investigations/cross-referencing/index.mdx b/docs/src/pages/users/investigations/cross-referencing/index.mdx
new file mode 100644
index 0000000000..4183f21d49
--- /dev/null
+++ b/docs/src/pages/users/investigations/cross-referencing/index.mdx
@@ -0,0 +1,247 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: Cross-referencing your data
+---
+
+import SchemaListing from '@components/SchemaListing.astro';
+
+# Uploading lists to cross-reference your data against Aleph datasets
+
+
Uploading your data to Aleph allows you to compare it against hundreds of other datasets such as sanctions lists, company registries, and more. This process is called cross-referencing. This page explains how to upload Excel or CSV files to Aleph in order to cross-reference them.
+
+Let's say you have a list of names of people or companies and you want to upload it to Aleph to see if there are matches between your names and other datasets to which you have access.
+
+## Preparing your data
+
+First of all it is important to keep in mind that your Excel or CSV file needs to be clean and well organized. What does this mean? It means that the columns containing the names of persons, companies or corporations should only contain that type of information.
+
+
+ Useful tips when working with Excel or CSV files in Aleph:
+
+ * Avoid columns with multiple properties
+ * Try to stick to one format, for example always use the same date or address format
+ * Avoid duplicates and provide additional identifiers.
+ * Where possible, add information like date of birth, aliases, addresses, e-mails, IDs
+
+
+### ✅ Good example
+
+Cleaned and well-organized columns for cross-referencing:
+
+| # | Full name | Position |
+| --- | --- | --- |
+| 1 | Maria Consuelo Ramírez Scaglia | Secretaria General de la Presidencia |
+| 2 | Diego José Montenegro López | Subsecretario General de la Presidencia |
+| 3 | Sergio Alejandro Rosal Morales | Subsecretario General Administrativo |
+| 4 | Claudia Isabel Mérida Pérez | Asesor Jurídico |
+| 5 | Omar Ricardo Barrios Osorio | Asesor Jurídico |
+| 6 | Ana Lucía del Aguila García | Asistente Despacho Superior |
+| 7 | Gabriela Vásquez Galindo de Escobar | Asistente Despacho Superior |
+
+### ❌ Bad example
+
+Table without proper data cleaning:
+
+| # | Full name | Position |
+| --- | --- | --- |
+| 1 | Maria Consuelo Ramírez Scaglia (check) | Secretaria General de la Presidencia |
+| 2 | Diego José Montenegro López | Subsecretario General de la Presidencia |
+| 3 | Diego José M. López | Subsecretario General de la Presidencia |
+| 4 | Claudia Isabel Mérida Pérez - CHECK | Asesor Jurídico |
+| 5 | 1245 - Omar Ricardo Barrios Osorio | Asesor Jurídico |
+| 6 | Ana Lucía del Aguila García | Asistente Despacho Superior |
+| 7 | Gabriela Vásquez Galindo de Escobar | Asistente Despacho Superior |
+
+Why is the table above a bad example? There are comments after people's names (row 1 and 4) and the same person is mentioned in different ways (row 2 and 3). The same can happen with companies. This is why it is important to check the list structure and information before uploading the sheet in Aleph.
+
+## Uploading your spreadsheet
+
+In order to cross-reference data from a spreadsheet, you first need to upload the spreadsheet to Aleph.
+
+
+
+ Go to **Upload documents** in your investigation:
+
+
+
+
+
+ Then find the file on our computer and upload it. In our case, the Excel file is called **Data - draft**:
+
+
+
+
+
+ Click on **Close** and wait for a few seconds.
+
+
+
+ Then go to the **Documents** section (left menu). There you will see the uploaded file:
+
+
+
+
+
+## Generating entities from your spreadsheet
+
+Once you have upload your spreadsheet to Aleph, you will need to generate entities from the rows in the spreadsheet. Once you have generated entities, you can then compare these entities to other datasets in Aleph.
+
+
+
+ Click on **Data - draft.xlsx**, the spreadsheet you just uploaded. You will see that the file has three sheets:
+
+
+
+
+
+ We must always select the sheet that contains the data we want to cross-reference against Aleph databases. In our case, the information is in the **PEP** sheet. By clicking there we will see a preview of the data in our list. As you can see in the image below, we are in View mode, although there are also the options Mentions and Generate entities, all of them located above our table:
+
+
+
+
+
+ Please select **Generate entities**. Aleph will take us to a new screen. Why? Because Aleph needs more details to understand what is on your sheet:
+
+
+
+
+
+ Then go to **1. Select entity types to generate** and click on **+ Add new object**. This is where we have to tell Aleph the type of information we have in our list. These are the available options:
+
+
+
+ Since the example contains a list of persons, we must select **Person**. If we have a list of companies, choose Company instead.
+
+
+ In case you have sheets that mix people and companies (and you have no way to distinguish between them), you can use the "Legal Entity" type.
+
+
+
+
+ When adding the object, in this case **Person**, the following box appears:
+
+
+
+ Now Aleph asks you to indicate which columns in our file have information relating to individuals. As we may not remember the names of the columns, we can scroll down and consult the information in our file before completing this section:
+
+
+
+ Another option is to click inside the **Keys** box, where it says **Select keys from available columns**:
+
+
+
+
+ Keys are features that make an entity (person, company etc) unique amongst others, and which help us distinguish entities one from another.
+
+
+ For this example, we select **Full name** and **Email**. Why didn't we choose the other columns? Because the best keys are columns that contain ID numbers, phone numbers, e-mail addresses or other information that allows us to uniquely identify an entity.
+
+
+
+ Next, we need to go to the preview of our table (below the box we just filled in) and specify the properties of the columns we will use for our x-ref. How do we do this? Under the column name, click on **Assign a property**.
+
+
+
+ In our case, the column **Full name** refers to **Person**.
+
+
+
+
+
+ For the **E-mail** column, we choose **Person** as the property and then **E-Mail**. The list of options is extensive, so don't forget to scroll down to see all the enabled categories:
+
+
+
+
+
+ Our table will now look like this:
+
+
+
+
+
+ Finally, scroll to the bottom of the page and click on **Generate entities**:
+
+
+
+
+
+ How do we know that everything went ok? Go to the top of the page and you will see the following message:
+
+
+
+
+
+## Cross-referencing
+
+Now that we have uploaded our list and adapted it to the Aleph model, you can finally cross-reference your data against other datasets!
+
+
+
+ Click on the name of our investigation at the top of the screen and go to **Cross-reference** in the left-hand menu:
+
+
+
+
+
+ You will see this screen:
+
+
+
+ Click on **Compute** and the data will start to be cross-referenced. Depending on the size of our list, this process may take more than a few minutes.
+
+
+
+ Once the cross-reference process finishes, we suggest waiting 2-3 minutes after the process is over so all matches are visible. Refresh your page to see all matches.
+
+
+
+{/* TODO Add short explanation of xref results page */}
diff --git a/docs/src/pages/users/investigations/entity-editor/index.mdx b/docs/src/pages/users/investigations/entity-editor/index.mdx
new file mode 100644
index 0000000000..a0de9553eb
--- /dev/null
+++ b/docs/src/pages/users/investigations/entity-editor/index.mdx
@@ -0,0 +1,109 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: 'Creating and editing entities'
+---
+
+# Creating and editing individual entities
+
+
Aleph provides an easy-to-use spreadsheet interface for creating, editing, linking, and deleting structured entities within an investigation workspace.
+
+## Creating entities
+
+While entities (names of people, companies, bank accounts, addresses, etc.) can be added to an investigation in bulk by creating network diagrams or via generation from an uploaded spreadsheet (see next section), it is also possible to create and edit entities individually.
+
+
+
+ To get started, go to **Entities** on the sidebar of any investigation page and select one of the options listed under **Add a new entity type**:
+
+
+
+ Let’s say you want to add a list of 5 people you are interested in checking if they appear in Aleph datasets. You just need to click on **Add new entity type** and select **Person**.
+
+
+
+ You will now see the following screen:
+
+
+
+ You can then start typing the full name of the person and other relevant data you might have such as **Nationality** and **Birth date**. The format for birth dates in Aleph is `YYYY-MM-DD`. In short, most properties, like a person's name or address, are free text values, but other properties, like birth date or nationality are restricted to date or country values, respectively. When you click to edit a cell, the table editor will display a slightly different editing input, depending upon the type of the property value you are editing.
+
+
+
+ Once you’ve entered data in a cell, press **Enter** to save the changes or click outside of the cell you have edited.
+
+
+
+ By clicking on the **Add property** button, you can select other data categories you want to add regarding people you are adding to the table. For example, you can add the person’s country of origin and addresses, among other available options:
+
+
+
+
+
+ To create another new entity using the table editor, either click the **Add new** button at the top of the table:
+
+
+
+ Or simply start typing in one of the cells of the bottom row of the table:
+
+
+
+
+
+
+## Editing entities
+
+Editing properties of an entity in the table editor works in much the same way as any spreadsheet editor like Microsoft Excel or Google Sheets.
+
+
+
+ Click in a cell or simply start typing in an active cell to edit its contents
+
+
+
+ Click outside of a cell or press **Enter** to finish editing a cell.
+
+
+
+ Copy and paste the value of a cell or multiple cells to duplicate values in other cells of the table.
+
+
+
+## Adding additional properties
+
+As mentioned above, each column in the table corresponds to a [property](/users/getting-started/key-terms#properties): an element connected to the person, company, contract, etc. in your table. For example: an ID, an address or an e-mail.
+
+By default, the table editor displays the most commonly-used properties of an entity type (e.g. name, nationality, and birth date for a person) as well as any properties that contain values. But each entity type has many more additional properties that can be added to the table. To add an additional property:
+
+
+
+ Scroll to the far right of the table.
+
+
+
+ Click the **Add a property** button.
+
+
+
+ Then select a property from the list.
+
+
+
+
diff --git a/docs/src/pages/users/investigations/manage-access/index.mdx b/docs/src/pages/users/investigations/manage-access/index.mdx
new file mode 100644
index 0000000000..6ef2e26036
--- /dev/null
+++ b/docs/src/pages/users/investigations/manage-access/index.mdx
@@ -0,0 +1,38 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: Manage access
+---
+
+# Manage access to your investigation
+
+
After creating your investigation workspace, access can easily be expanded to any co-collaborators you would like to include in your investigation.
+
+
+
+ Click on the **Gear icon** in the top right corner of the screen to open the **Access control** settings. In the dropdown menu, click on **Share**.
+
+
+
+
+
+ The access control window will then appear, allowing you to share your investigation workspace with other individuals or groups you are a part of.
+
+ Keep in mind that an individual must first have a user account on Aleph to receive access to an investigation workspace. Sharing an investigation with a group means that any user in that group will receive the access you are granting.
+
+ * **View access** allows you to browse the contents of an investigation workspace but not create, edit, or delete anything contained within.
+ * **Edit access** allows you full privileges to modify its contents.
+
+
+
+
+
+ Click on **Save changes**.
+
+
+
diff --git a/docs/src/pages/users/investigations/network-diagrams/index.mdx b/docs/src/pages/users/investigations/network-diagrams/index.mdx
new file mode 100644
index 0000000000..3b9140268f
--- /dev/null
+++ b/docs/src/pages/users/investigations/network-diagrams/index.mdx
@@ -0,0 +1,56 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: Drawing network diagrams
+---
+
+# Drawing network diagrams
+
+
Over the course of an investigation, it is sometimes helpful to sketch out a network of involved parties so that you can understand who is linked to what.
+
+Network diagrams let you illustrate webs of people, companies and other entities involved in an investigation in a visual diagram of actors and connections. It is important to note that all entities and relationships that you add to your diagram will be automatically added to the investigation in which the diagram is contained.
+
+## Creating a network diagram
+
+
+
+ To get started, click the **Network diagrams** button in the sidebar on any investigation page, and then the **New diagram** button.
+
+
+
+
+
+ You will now be able to map people, companies, and the relationships between them, which will be saved within your Aleph account, and only be visible to you and whomever you choose to share the diagram with.
+
+
+ {/* TODO Short explanation of diagram screen */}
+
+
+## Adding entities and links
+
+
+
+ To add people, companies, or other entities to your diagram, click the **Add entities** icon in the toolbar on the left side of the screen, or double click anywhere in the graph area.
+
+
+
+
+ Any entities (people, companies, etc) and relationships you create within a diagram will automatically be added to the investigation in which your diagram was created. This allows you to [cross-reference](/users/investigations/cross-referencing#cross-referencing) the lists of people, companies, and other entities you have created in your diagrams with data from across the rest of Aleph.
+
+
+
+
+ Once you have created multiple entities in your diagram, you can connect them by clicking the **Add links** icon.
+
+
+
+
+
diff --git a/docs/src/pages/users/investigations/overview/index.mdx b/docs/src/pages/users/investigations/overview/index.mdx
new file mode 100644
index 0000000000..9a56571175
--- /dev/null
+++ b/docs/src/pages/users/investigations/overview/index.mdx
@@ -0,0 +1,35 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: About investigation workspaces
+---
+
+# About investigation workspaces
+
+
Investigations workspaces within Aleph allow you to upload, edit, and organize data or documents related to a project or topic of interest – and they can be shared with other users or access groups within Aleph. Uploading documents to investigation workspaces has many advantages:
+
+## Collaboration
+Uploading documents to Aleph provides an easy way to share access to large sets of documents with your colleagues and collaborators.
+
+## Text extraction
+Aleph extracts text from images and PDF files, allowing you to search for key terms of interest in files that would otherwise prove difficult to search.
+
+
+
+## Names, phone numbers, addresses, …
+Aleph is trained to recognize and extract mentions of people, companies, phone numbers, addresses, and IBAN numbers contained in documents you upload, allowing you to easily find other documents containing matching mentions.
+
+
+
+## E-mail archives
+When uploading a set of e-mails, Aleph automatically extracts structured information like the sender, receiver, cc'ed entities, and attachments, making it easy to filter the uploaded data for messages sent from or to a specific person.
+
+
diff --git a/docs/src/pages/users/investigations/uploading-documents/index.mdx b/docs/src/pages/users/investigations/uploading-documents/index.mdx
new file mode 100644
index 0000000000..1ec5c37040
--- /dev/null
+++ b/docs/src/pages/users/investigations/uploading-documents/index.mdx
@@ -0,0 +1,47 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: Uploading documents
+---
+
+# Uploading documents
+
+
Learn about how to add files to Aleph, and the ways that Aleph can help to extract meaningful information from documents.
+
+The most common types of data to import into an investigation workspace in Aleph are PDFs, E-mail archives, or Word documents. Uploading these documents to Aleph allows you to keep track of evidence gathered over the course of an investigation and to share it as needed with your colleagues.
+
+Most importantly, uploading files to Aleph makes it easier to search their contents, even if text is hidden in images or other unstructured formats.
+
+
+
+ To upload documents, click on **Upload documents** in your investigation workspace. You will see the following screen:
+
+
+
+ If you want to upload one or more documents, click on **Browse**. If you want to upload folders, select **Click here** instead (light blue text).
+
+
+
+ After selecting documents or folders, start the upload by clicking the **Upload** button.
+
+
+
+ It will take a little while to process your uploaded files. While Aleph is processing your files, a small spinning indicator is displayed next to the name of your investigation workspace at the top of your screen. Hover over the name of your investigation workspace to see detailed progress information.
+
+
+
+
+
+ When the upload is finished, you should now see your files in the **Documents** section of the investigation workspace. You may cease frolicing.
+
+
+
+
diff --git a/docs/src/pages/users/search/advanced/index.mdx b/docs/src/pages/users/search/advanced/index.mdx
new file mode 100644
index 0000000000..eda7f27a78
--- /dev/null
+++ b/docs/src/pages/users/search/advanced/index.mdx
@@ -0,0 +1,129 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: Advanced search
+---
+
+# Advanced search options
+
+
Beyond a simple keyword search, Aleph supports many more complex search operations to find matches based on spelling variations, proximity to other terms, and much more.
+
+To view the advanced options available in Aleph, click the advanced search button next to the search bar. A pop-up will appear explaining each of the available operations.
+
+
+
+Each of the options in the pop-up allows you to enter a term or terms. Once you have entered a term, clicking **Search** executes a search with your updated advanced option. You will notice that each time you enter a new advanced option, the keywords in the main search bar will change, indicating the option you've selected.
+
+We will go through each of the advanced search operations in more detail below.
+
+## Finding an exact phrase or name
+
+By default, Aleph returns matches that include all of your keywords first, followed by matches that might only include one of your keywords.
+
+For example, if you type the keywords:
+
+```
+Vladimir Putin
+```
+
+Aleph will return all matches that have the words "Vladimir" and "Putin," followed by matches that have either "Vladimir" or "Putin" but not both, in them. Depending on your needs, this might not be ideal.
+
+{/*TODO Check if the above is still correct.*/}
+
+If you want Aleph to only return matches that have exactly "Vladimir Putin", then you should put quotations around those two keywords:
+
+```
+"Vladimir Putin"
+```
+
+
+ A quoted search like "Vladimir Putin" returns results that contain all terms in the exact same order. However, Aleph applies some normalization (like transliteration) of the keyword and documents, so searching for "Vladimir Putin" would also return results that contain "Владимир Путин" (and vice versa).
+
+
+
+
+Instead of using quotes, you can also do this by writing "Validimir Putin" in the **This exact word/phrase** field in the advanced search options:
+
+
+
+## Allow for variations in spelling
+
+Sometimes a name can be spelled many different ways or even misspelled many different ways. One way to solve this problem is to simply type each variation in the search form:
+
+```
+Владимир Путин Poutine Wladimir Путин Путину Путином
+```
+
+You might capture all the variations you want, but you also might miss some by accident. Another way to tell Aleph to look for variants of a name is to use the ~ operator:
+
+```
+Putin~2
+```
+
+What this translates to: Give me matches that include the keyword "Putin", but also matches that include up to any 2 letter variations of "Putin". These variations include adding, removing, and changing a letter.
+
+{/*TODO Clarify and add example*/}
+
+
+
+Using this operator with too high a number (greater than 3, for example) will cause slow searches and may return too many false results.
+
+{/* TODO The max value for fuzzy searches is 2. */}
+
+## Search for words that should be in proximity to each other
+
+If you do not want to find a precise keyword, but merely specify that two words are supposed to appear close to each other, you might want to use a **proximity search**, which also uses the **~** operator. This will try to find all the requested search keywords within a given distance from each other. For example, to find matches where the keywords Trump and Putin are ten or fewer words apart from each other, you can formulate the search as:
+
+```
+"Trump Putin"~10
+```
+
+## Including and excluding combinations of keywords
+
+You can tell Aleph to find matches to multiple keywords in a variety of ways or combinations, otherwise known as a composite search.
+
+To tell Aleph that a keyword must exist in all resulting matches, use a **+** operator. Similarly, to tell Aleph that a keyword must not exist in any of the resulting matches, use - operator.
+
+```
++Trump -Putin
+```
+
+This translates to: Give me all matches in which each match must include the keyword Trump and must definitely not include the keyword Putin.
+
+You can take these combinations a step further using the **AND** operator or the **OR** operator.
+
+```
+Trump AND Putin
+```
+
+This translates to: Give me all matches in which each match must contain both the keywords Trump and Putin, but don't return any matches that only contain just one of those keywords.
+
+```
+Trump OR Putin
+```
+
+This translates to: Give me all matches in which each match may contain the keywords Trump or Putin or both.
+
+You can build on these searches even further like so:
+
+```
++Trump AND (Salman OR Putin) -South Korea
+```
+
+This translates to: Give me all matches in which each match must contain the keyword Trump and must contain either the keyword Salman or the keyword Putin, but must not contain the keywords South Korea.
+
+## Putting it all together
+
+You can combine any of the methods supported by Aleph in many combinations to create some very explicit search rules. The complexity, of course, depends on your needs.
+
diff --git a/docs/src/pages/users/search/basics/index.mdx b/docs/src/pages/users/search/basics/index.mdx
new file mode 100644
index 0000000000..a1a78db582
--- /dev/null
+++ b/docs/src/pages/users/search/basics/index.mdx
@@ -0,0 +1,149 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: Searches, filters, and alerts
+---
+
+# How to do searches, filter, and set alerts
+
+
The section below explains how to carry out a basic search, view, bookmark, and filter your results, and create e-mail alerts for terms of interest.
+
+
+
+## Basic search
+
+Aleph's site-wide search bar is located at the top of any page on the site. This search bar allows you to search across all datasets you have access to in Aleph. It is comprised of several distinct features:
+
+
+
+{/*TODO Use search bar/box consistently*/}
+In the search bar, you simply enter the term you are searching for. In this example, we are searching for Vladimir Putin.
+
+
+
+When you execute a search, the results of your search are listed below the search bar. As you scroll, more of the results will be displayed, until you reach the end of the list. The result count lets you know how many results in total were found matching your term:
+
+
+
+
+ Keep in mind that Aleph limits the number of search results you can download based on the number of results and document size
+
+
+With each result, **yellow highlighted** text indicates the context in which your keyword was found. This allows you to eyeball the results and understand why they were returned based on the terms of your search. This can also help you decide if you need to either expand or narrow down your search.
+
+## Previewing a result
+
+
+
+On the screen, click on the possible hits that appear under the **Name** column to explore a preview of a possible match. Go to the **Expand** button to explore the result in full within the specific dataset where your hit appears. The **Download** button allows you to download the file you are previewing.
+
+{/*TODO Add warning about downloads*/}
+
+You can also use the **Bookmark** button to keep track of all your interesting files and entities you explore in Aleph and you can view your bookmarked files and entities by clicking on **Bookmarks** in the top bar.
+
+{/*TODO Add link to bookmarks article*/}
+
+
+ Bookmarks are still an experimental feature and are only stored in your browser. If you delete your browsing data or switch devices, you may lose access to your bookmarks. You can always download a copy of your bookmarks in the bookmarks section.
+
+
+The **X** button closes the preview, returning you to the main results view.
+
+## Filtering results
+
+Aleph provides built-in options to filter your search results by country, type of data you are interested in (document, company entity, web page, image, video, etc.), entity type, specific datasets, e-mail addresses, phone numbers, bank accounts, names, language, and so on. You can view all of the built-in options from the left-hand side menu following the execution of a search.
+
+
+
+Each filter type provides a list of categories that allow you to select exactly what you most want to see. In addition, options are **sorted from most prevalent to least** with the quantity of occurrences beside it. This number can be quite useful, as it shows the most common options first, giving you an indicator of where to start filtering to narrow down your search.
+
+When you **select a filter value**, a button appears above the search results indicating that the results are being filtered by that option. If you want to remove the filter, it is easy enough to click on the **X** in the button to remove it.
+
+In the example below, we are searching for results including both Trump AND Putin and then filtering by the entity type: **e-mail**.
+
+
+
+### Filtering by date
+
+When selecting the **Dates** filter option, you are able to see a distribution showing the number of results per year. You can use this filter in two ways. By selecting a year, you are able to further filter your search results by month, as well as by day, by clicking on the corresponding bars, depending on your needs. If you are interested in searching throughout a range of dates, you can click and drag bars across the distribution to select a range.
+See the example below on how to filter by a specific date using the Dates filter.
+
+
+
+See the example below on how to filter by a range of dates using the Dates filter.
+
+
+
+
+## Creating alerts
+
+Whenever you search a term in Aleph, you can use the **Create alert** button to generate an alert based on the keywords and operators entered in the search bar. Keep in mind that you will not see the Alert button by default in the search bar. It only gets activated once you write a word or term there:
+
+
+
+Once selected, you will receive e-mail notifications as well as see alerts in the [notifications tab](/users/getting-started/homepage#notifications) when new data is added to Aleph that meet the requirements of your search.
+
+
+ You will only receive alerts for datasets that you have access to.
+
+
+{/*TODO Clarify wording notifications/alerts*/}
+{/*TODO How to delete alerts*/}
+
+## Download a document
+
+Within Aleph, you are able to download individual documents. When clicking on an individual search result, in the pop-up window that appears on the right-hand side of your screen, you simply click on **Download** and the download will start.
+
+
+ When downloading a file, you are downloading a source file. Source files can contain viruses and code that notify the originator when you open them. For sensitive data, we recommend only opening source files on a computer that is permanently disconnected from the internet.
+
+
+
+
+## Export multiple search results
+
+If you wish to export all search results, in the top center of the screen, you simply click on the **Export** button and the export will start. Depending on the number of search results, preparing your export may take some time.
+
+
+
+You can track the status of your exports by navigating to the Aleph homepage followed by clicking on **Exports** from the left-hand side menu or by clicking on your profile name in the upper right-hand corner followed by clicking on **Exports**.
+
+For more detailed information refer to [How to navigate the Aleph homepage](/users/getting-started/homepage).
+
+{/*TODO Add warning about size limitations */}
diff --git a/docs/src/pages/users/search/datasets/index.mdx b/docs/src/pages/users/search/datasets/index.mdx
new file mode 100644
index 0000000000..7191183148
--- /dev/null
+++ b/docs/src/pages/users/search/datasets/index.mdx
@@ -0,0 +1,48 @@
+---
+layout: '@layouts/UsersLayout.astro'
+title: Searching within specific datasets
+---
+
+# How to find and search within specific datasets
+
+
In Aleph, datasets and investigations serve as the primary containers for organizing and managing collections of documents and entities. Any document or entity in Aleph must be a part of either a dataset or an investigation. In this section, we will explain how to search within a specific dataset.