sa7mon · sa7mon · Apr 23, 2021 · Sep 30, 2019 · Sep 30, 2019 · Sep 30, 2019
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,4 @@
+[run]
+omit =
+    venv*
+    tests/TestUtils.py
diff --git a/.gitignore b/.gitignore
@@ -1,10 +1,24 @@
-venv/
-venv2.7/
+# IDE
 .idea/
+.vscode/
+
+# Virtualenv
+venv*/
+
+# Cache
 __pycache__
 *.pyc
 .cache/
+
+# Testing
 .pytest_cache
-list-buckets/
+test/
+.coverage
+htmlcov/
+temp-*/
 buckets.txt
-test/
+
+# Pip build
+build/
+dist/
+S3Scanner.egg-info/
diff --git a/.travis.yml b/.travis.yml
@@ -1,16 +1,28 @@
 language: python
-matrix:
+jobs:
   include:
-  - python: '3.6'
-    env:
-      - secure: QCHu2+V5MO4OtZfBmB9yZVof0NeHK/3CGmErdK/4ICnvfkf4wteYKdDgFsZyzt/ns3BCuP3dZvXEU9p4EEudjMEv66RvPyr+q29JhK42a+HjHP7XAHBMLEENkihtBtrsnfr7NrSgPtfLkG4XaJVyn9mPCvo4JuDotldC4jlAEJqkByKbh5koI4533WOG36pmvO7jcsuqm3+ii7J9PYCRqqbqjvWjt1eJx8ITiBPbJ8+1e32HGWKSMJMJhe2RlxuhglQmOMlZDKrtQ/5u+pgMWtItjP798zJRyKEG8zrt+PM2vgYBeHB9rFMqCmtEO68zpB+HhYXb21R7qkDSTocL6uMKkR1cYplKwilFTLMutfUfCv3qRT77ciW8GGfHEbB7mukeeu8Jvz2lphFvbiRPiReQlWbmn0zJaSOW1SbqrGYrAy6WM4IyJR4v1sDPXZHUq6wYFywKqpbA4sF59RAVrPvMu/Z8Zhdol+P5CV6FZnTr1fz3JTEea/CMnbExKQfFUNgvAd11WncIyrxV0Pr26iR6HD8ajRQh3Uk8cvYtUZnnvTvSIF2Rp913RdJVaBjy1mZ/IarkhZIGp4F6y0892Ok6bRXkMsDLxkMMWWRWWUYgM/8UY9tckv4mBPDi8J9u2UAjYpkgxdyweCjI2F3aRBbRMiFZJPmepGbiUipk0zg=
-      - secure: Phv6W+P19/wJQfY2frJNuqldZc4snt8gUaI8ZtXxTLBFYaUpsbM4Pl2lJErp+W4k0zZ9I0i6EKs+WWj4jLkmg5tauXL7O3C4QNK3u5RAD154bcG1up8U5XPrLvz0ZRzvbAJilQQpjLrMyLO7GXaZIzZIN4MaUCuTTlvKls5fCp3Q2UJZdCpW94zlyFKInkk5Ox1wZsCbjptG5i6eGm/Q+g/ObuyfMMykUISjP7/BElb3lbVnIe2MCWQxoNlt1d0T6oaUSkIbnaYD384uEkYn5bAlEoOyUlurgMyifPWoBI4vLJHBj8votdgjxcgThGzqjfsB29U/eaaP3VUc77CeBLMGwoYzqIPcoicUBkK5GowvIXkukoUqN24JK5kTROv/aYOdVqoLCrT8fYruVdS0kkJs3DZbBoMDaWte36PhvJouh7MPaWgdm1ZcfLP2juLe5UGaiuN5iY95L/XBXbrnd6+adJYrJd7TYX6+zJE2trmwaxqZCELQtUVUcrYJ85KljjQS7QJQTE1uXQRlPCzGQlhziC8ePJI1zOuHQV7xZWbi04Jg6Pe9zE3UXnBU6KvB/q3fLdrFT5F8mqI2lUXFl+kq28sucqgUMM1rfmW30OKkyHfiRuTZL43Img0Xduql++DgLz6WweFhXf8oH18b8XuV7KUZz5FFJgjs86FM8AE=
   - python: '3.6'
   - python: '3.7'
+  - python: '3.8'
+  - python: '3.9'
+    env:
+      - secure: "JShcKAHn4y57mTHDIV5+8dTRjE2cREJSswXAxhFf8jha+r58zF/uBfgXapzNh9u+dpvbVjF/N0/KxREubMTd4fduYTsxMOXyqENHnq7kVmRK6HXAAnM75JZzl1sonlHsIHHXxv45SuwYWX/fk6aMeBmkGukuvM8DGi4BEBzv0CnzEUmHlb5ZPKmQteemjhbn2d3yKPKagcieeDbSRhevGKPPmfnt0TqzpF/xrbtIL05yC+038Tesa0mZqV/HBrfZgSEtcMydIhbszhDjBwC3nzhhiC8AQJ8JGRPqR3nfTZRrHA0QMT3hr8XGpLouphvpDDwiotmOTRsGiBfONX+b2JDTx989eswIXmBsdua3pxjUNuLVTiRjl63+6zJSvT3mrJ1cZMRJPvbqYTY+mvckSMeDQv4oFZeD8QCD+z8zLa39GYfKBnapo0s+rvvxYyiVNZ9HQ1MExJyVleJWRMlmKtuNUHzHaCq+B8omcGZxhEfX4dVQ/RHwNRwkKbdUKOZy4muardhYorhVO+eLt4+bAipk8BEAXvIBwaAqbIN3+01a8TbTGKkxJUTllkf2Y7wFeF6IPtxvfpJ6Bgj4BNSpDrR/eoyIodG42J6Qdl4aK6/RQbI9vzUQ8yoSxQxzHHFZeclU2Qe5KZem3ztbexkiYB+Mv7oV/rr1LGixvbBsLzI="
+      - secure: "SLjBJJsmtbHZGwmZHHJTYk0qmlS6kcbur1SMM70+n/UEp55hAFo6Ae/n75G4RR0bVIVkgrJp3ZE9V/wZKVbOOaUaepyjZXfgRBjL/zBYjFgxgQhrLis3Bg+lR6qBoWifm/mfM+mUqHLDqelSbvpgE/oZLeM9vuYBYvI9LIZSeM3C6m+4ytKoayUgggq87lQRDr9d/YPpAZEYnT13mAqkd3zbovjLAEtALx6BOg5xZv7bHCx5WS5gz79CA+jFRjWU9q4ng5zyCERWFOeTcCYAHjxKXYOJYew8N/NYA2PFd+BiedQRHuIJAHg/auofchBewmtfHG6rgMZSuE+jzl1aB344zwpVJocR09kBXi6tk9KiASTrZMSHf31LEJFwciFnSsCTVQG7kVL0NdjZBEGO7zE1u5c4dxYEctDVvCz/kmH7hl70zajot9cYihh8VLvLwpGBYepTf2a+vhRwdxeZ81KuI3SeOBqNJwyT6wZMw9AfEOmK9LqyS9vBqusujwua+W/DDeqYFo99HkS2uMdX4/IfAB5DDhVakMrff8rrUuf1K1H6rtV7qckOHDET+wdjqymZkPD/mjjW+ibAattls4cZU3I7NRVsnNiZmXAT410A92y6JEiZPuG1djz/57yrqQ3S4KzVhgBq1t5WJRc84dUKCbYnwY4fDL4BH8lkync="
+  - name: "Python: 3.6"
+    os: windows
+    language: shell
+    before_install:
+      - choco install python --version=3.6.8
+      - python -m pip install -U pip setuptools
+    env: PATH=/c/Python36:/c/Python36/Scripts:$PATH
+    cache:
+      directories:
+        - $LOCALAPPDATA/pip/Cache
+
 cache: pip
 install:
 - pip install -r requirements.txt
 script:
-- pytest ./test_scanner.py -s
+- pytest -s
 notifications:
-  email: false
+  email: false
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,6 @@
-FROM python:3-alpine
+FROM python:3.8-alpine
 COPY . /app
 WORKDIR /app
-RUN pip install -r requirements.txt
-ENTRYPOINT ["python", "s3scanner.py"]
+RUN pip install boto3
+RUN pip install .
+ENTRYPOINT ["python", "-m", "S3Scanner"]
diff --git a/LICENSE.txt → LICENSE b/LICENSE.txt → LICENSE
diff --git a/README.md b/README.md
@@ -1,134 +1,136 @@
 # S3Scanner
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Build Status](https://travis-ci.org/sa7mon/S3Scanner.svg?branch=master)](https://travis-ci.org/sa7mon/S3Scanner)
 
-A tool to find open S3 buckets and dump their contents :droplet:
-
-![1 - s3finder.py](https://user-images.githubusercontent.com/3712226/40662408-e1d19468-631b-11e8-8d69-0075a6c8ab0d.png)
-
-### If you've earned a bug bounty using this tool, please consider donating to support it's development
-
-[![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=XG5BGLQZPJ9H8)
+A tool to find open S3 buckets and dump their contents💧
 
+<img src="https://user-images.githubusercontent.com/3712226/115632654-d4f8c280-a2cd-11eb-87ee-c70bbd4f1edb.png" width="85%"/>
 
 ## Usage
-
 <pre>
-usage: s3scanner [-h] [-o OUTFILE] [-d] [-l] [--version] buckets
-
-#  s3scanner - Find S3 buckets and dump!
-#
-#  Author: Dan Salmon - @bltjetpack, github.com/sa7mon
+usage: s3scanner [-h] [--version] [--threads n] [--endpoint-url ENDPOINT_URL] [--endpoint-address-style {path,vhost}] [--insecure] {scan,dump} ...
 
-positional arguments:
-  buckets               Name of text file containing buckets to check
+s3scanner: Audit unsecured S3 buckets
+           by Dan Salmon - github.com/sa7mon, @bltjetpack
 
 optional arguments:
   -h, --help            show this help message and exit
-  -o OUTFILE, --out-file OUTFILE
-                        Name of file to save the successfully checked buckets in (Default: buckets.txt)
-  -d, --dump            Dump all found open buckets locally
-  -l, --list            Save bucket file listing to local file: ./list-buckets/${bucket}.txt
   --version             Display the current version of this tool
+  --threads n, -t n     Number of threads to use. Default: 4
+  --endpoint-url ENDPOINT_URL, -u ENDPOINT_URL
+                        URL of S3-compliant API. Default: https://s3.amazonaws.com
+  --endpoint-address-style {path,vhost}, -s {path,vhost}
+                        Address style to use for the endpoint. Default: path
+  --insecure, -i        Do not verify SSL
+
+mode:
+  {scan,dump}           (Must choose one)
+    scan                Scan bucket permissions
+    dump                Dump the contents of buckets
 </pre>
 
-The tool takes in a list of bucket names to check. Found S3 buckets are output to file. The tool will also dump or list the contents of 'open' buckets locally.
-
-### Interpreting Results
-
-This tool will attempt to get all available information about a bucket, but it's up to you to interpret the results.
+## Support
+🚀 If you've found this tool useful, please consider donating to support its development
 
-[Settings available](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/set-bucket-permissions.html) for buckets:
-* Object Access (object in this case refers to files stored in the bucket)
-  * List Objects
-  * Write Objects
-* ACL Access
-  * Read Permissions
-  * Write Permissions
-
-Any or all of these permissions can be set for the 2 main user groups:
-* Authenticated Users
-* Public Users (those without AWS credentials set)
-* (They can also be applied to specific users, but that's out of scope)
-
-**What this means:** Just because a bucket returns "AccessDenied" for it's ACLs doesn't mean you can't read/write to it.
-Conversely, you may be able to list ACLs but not read/write to the bucket
+[![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=XG5BGLQZPJ9H8)
 
+[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/B0B54D93O)
 
 ## Installation
-  1. (Optional) `virtualenv venv && source ./venv/bin/activate`
-  2. `pip install -r requirements.txt`
-  3. `python ./s3scanner.py`
-
-(Compatibility has been tested with Python 2.7 and 3.6)
 
-### Using Docker
+```shell
+pip3 install s3scanner
+```
 
- 1. Build the [Docker](https://docs.docker.com/) image:
+or via Docker:
 
- ```bash
-sudo docker build -t s3scanner https://github.com/sa7mon/S3Scanner.git
+```shell
+docker build . -t s3scanner:latest
+docker run --rm s3scanner:latest scan --bucket my-buket
 ```
 
- 2. Run the Docker image:
+or from source:
 
- ```bash
-sudo docker run -v /input-data-dir/:/data s3scanner --out-file /data/results.txt /data/names.txt
+```shell
+git clone git@github.com:sa7mon/S3Scanner.git
+cd S3Scanner
+pip3 install -r requirements.txt
+python3 -m S3Scanner
 ```
-This command assumes that `names.txt` with domains to enumerate is in `/input-data-dir/` on host machine.
+
+## Features
+
+* ⚡️ Multi-threaded scanning
+* 🔭 Supports tons of S3-compatible APIs
+* 🕵️‍♀️ Scans all bucket permissions to find misconfigurations
+* 💾 Dump bucket contents to a local folder
+* 🐳 Docker support
 
 ## Examples
-This tool accepts the following type of bucket formats to check:
-
-- bucket name - `google-dev`
-- domain name - `uber.com`, `sub.domain.com`
-- full s3 url - `yahoo-staging.s3-us-west-2.amazonaws.com` (To easily combine with other tools like [bucket-stream](https://github.com/eth0izzle/bucket-stream))
-- bucket:region - `flaws.cloud:us-west-2`
-
-```bash
-> cat names.txt
-flaws.cloud
-google-dev
-testing.microsoft.com
-yelp-production.s3-us-west-1.amazonaws.com
-github-dev:us-east-1
-```
-
-1. Dump all open buckets, log both open and closed buckets to found.txt
-
-	```bash
-	> python ./s3scanner.py --include-closed --out-file found.txt --dump names.txt
-	```
-2. Just log open buckets to the default output file (buckets.txt)
-
-	```bash
-	> python ./s3scanner.py names.txt
-	```
-3. Save file listings of all open buckets to file
-    ```bash
-    > python ./s3scanner.py --list names.txt
-
-    ```
-
-## Contributing
-Issues are welcome and Pull Requests are appreciated. All contributions should be compatible with both Python 2.7 and 3.6.
-
-|    master    |    [![Build Status](https://travis-ci.org/sa7mon/S3Scanner.svg?branch=master)](https://travis-ci.org/sa7mon/S3Scanner)    |
-|:------------:|:-------------------------------------------------------------------------------------------------------------------------:|
-| enhancements | [![Build Status](https://travis-ci.org/sa7mon/S3Scanner.svg?branch=enhancements)](https://travis-ci.org/sa7mon/S3Scanner) |
-|     bugs     |     [![Build Status](https://travis-ci.org/sa7mon/S3Scanner.svg?branch=bugs)](https://travis-ci.org/sa7mon/S3Scanner)     |
-
-### Testing
-* All test are currently in `test_scanner.py`
-* Run tests with in 2.7 and 3.6 virtual environments.
-* This project uses **pytest-xdist** to run tests. Use `pytest -n NUM` where num is number of parallel processes.
-* Run individual tests like this: `pytest -q -s test_scanner.py::test_namehere`
-
-### Contributors
+
+* Scan AWS buckets listed in a file with 8 threads
+  ```shell
+  $ s3scanner --threads 8 scan --buckets-file ./bucket-names.txt
+   ```
+* Scan a bucket in Digital Ocean Spaces 
+  ```shell
+  $ s3scanner --endpoint-url https://sfo2.digitaloceanspaces.com scan --bucket my-bucket
+  ```
+* Dump a single AWS bucket
+  ```shell
+  $ s3scanner dump --bucket my-bucket-to-dump
+  ```
+* Scan a single Dreamhost Objects bucket which uses the vhost address style and an invalid SSL cert
+  ```shell
+  $ s3scanner --endpoint-url https://objects.dreamhost.com --endpoint-address-style vhost --insecure scan --bucket my-bucket
+  ```
+
+## S3-compatible APIs
+
+`S3Scanner` can scan and dump buckets in S3-compatible APIs services other than AWS by using the
+`--endpoint-url` argument. Depending on the service, you may also need the `--endpoint-address-style`
+or `--insecure` arguments as well. 
+
+Some services have different endpoints corresponding to different regions
+
+**Note:** `S3Scanner` currently only supports scanning for anonymous user permissions of non-AWS services
+
+| Service | Example Endpoint | Address Style | Insecure ? |
+|---------|------------------|:-------------:|:----------:|
+| DigitalOcean Spaces (SFO2 region) | https://sfo2.digitaloceanspaces.com | path | No |  
+| Dreamhost | https://objects.dreamhost.com | vhost | Yes |
+| Linode Object Storage (eu-central-1 region) | https://eu-central-1.linodeobjects.com | vhost | No |
+| Scaleway Object Storage (nl-ams region) | https://s3.nl-ams.scw.cloud | path | No |
+| Wasabi Cloud Storage | http://s3.wasabisys.com/ | path | Yes |
+
+📚 Current status of non-AWS APIs can be found [in the project wiki](https://github.com/sa7mon/S3Scanner/wiki/S3-Compatible-APIs)
+
+## Interpreting Results
+
+This tool will attempt to get all available information about a bucket, but it's up to you to interpret the results.
+
+[Possible permissions](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/set-bucket-permissions.html) for buckets:
+
+* Read - List and view all files
+* Write - Write files to bucket
+* Read ACP - Read all Access Control Policies attached to bucket
+* Write ACP - Write Access Control Policies to bucket
+* Full Control - All above permissions
+
+Any or all of these permissions can be set for the 2 main user groups:
+* Authenticated Users
+* Public Users (those without AWS credentials set)
+* Individual users/groups (out of scope of this tool)
+
+**What this means:** Just because a bucket doesn't allow reading/writing ACLs doesn't mean you can't read/write files in the bucket. Conversely, you may be able to list ACLs but not read/write to the bucket
+
+## Contributors
 * [Ohelig](https://github.com/Ohelig)
 * [vysecurity](https://github.com/vysecurity)
 * [janmasarik](https://github.com/janmasarik)
 * [alanyee](https://github.com/alanyee)
+* [klau5dev](https://github.com/klau5dev)
 * [hipotermia](https://github.com/hipotermia)
 
 ## License
-License: [MIT](LICENSE.txt) https://opensource.org/licenses/MIT
+
+MIT