diff --git a/.dockerignore b/.dockerignore
index 3c3996eb4c56..f6c638b0a221 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,9 +1,13 @@
-Dockerfile
-.travis.yml
-.gitignore
-demo/etc
-tox.ini
-.git/*
-.tox/*
-debian/matrix-synapse/
-debian/matrix-synapse-*/
+# ignore everything by default
+*
+
+# things to include
+!docker
+!scripts
+!synapse
+!MANIFEST.in
+!README.rst
+!setup.py
+!synctl
+
+**/__pycache__
diff --git a/changelog.d/5564.misc b/changelog.d/5564.misc
new file mode 100644
index 000000000000..e209cdcc29b5
--- /dev/null
+++ b/changelog.d/5564.misc
@@ -0,0 +1 @@
+Reduce the amount of stuff we send in the docker context.