Feat/better deployment workflow

CONTRIBUTING.md

@@ -1,5 +1,14 @@
 Hello you,
 
+## Install the scrapper without Docker
+
+We do recommend the use of [pipenv][1] to install the whole python environment
+
+- [Install pipenv][2]
+- `pipenv --python 2.7 install`
+
+You should be ready to go.
+
 ## Run the tests
 
 To run the full test suite, run `./docsearch test`. Note that it requires that
@@ -11,16 +20,39 @@ If you are Algolia employee and want to manage a DocSearch account,
 you'll need to add the following variables in your `.env` file:
 
 ```
+APPLICATION_ID=
+API_KEY=
+APPLICATION_ID_PROD=
+API_KEY_PROD=
+HELP_SCOUT_API_KEY=
+NB_HITS_MAX=
+
 WEBSITE_USERNAME=
 WEBSITE_PASSWORD=
 SLACK_HOOK=
 SCHEDULER_USERNAME=
 SCHEDULER_PASSWORD=
+MONITORING_API_KEY=
+PUBLIC_CONFIG_FOLDER=<path to your docsearch-scrapy directory> + /configs/public
+PRIVATE_CONFIG_FOLDER=
+
+BASE_INTERNAL_ENDPOINT=
+INTERNAL_API_AUTH=
+
 DEPLOY_KEY=
 ```
 
 Ping the @docsearch team to get access to those credentials.
 
-The CLI will then have more commands for you to run. 
+## Ready to deploy
+
+Clone the configurations repo in your docsearch-scraper directory. Command to run from the scraper root:
+
+```bash
+git clone git@github.com:algolia/docsearch-configs.git configs/public
+```
 
+The CLI will then have more commands for you to run.
 
+[1]: https://github.com/pypa/pipenv
+[2]: https://pipenv.readthedocs.io/en/latest/install/#installing-pipenv

Pipfile

@@ -0,0 +1,53 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+algoliasearch = "==1.13.0"
+"asn1crypto" = "==0.22.0"
+attrs = "==17.2.0"
+certifi = "==2017.7.27.1"
+cffi = "==1.11.0"
+chardet = "==3.0.4"
+click = "==6.7"
+constantly = "==15.1.0"
+cryptography = "==2.0.3"
+cssselect = "==1.0.1"
+"enum34" = "==1.1.6"
+future = "==0.16.0"
+hyperlink = "==17.3.1"
+idna = "==2.6"
+incremental = "==17.5.0"
+ipaddress = "==1.0.18"
+lxml = "==4.0.0"
+ndg-httpsclient = "==0.4.3"
+ordereddict = "==1.1"
+parsel = "==1.2.0"
+"pyasn1" = "==0.3.5"
+"pyasn1-modules" = "==0.1.4"
+pycparser = "==2.18"
+pyperclip = "==1.5.27"
+python-dotenv = "==0.7.1"
+queuelib = "==1.4.2"
+ratelimit = "==1.4.1"
+requests = "==2.18.4"
+requests-file = "==1.4.2"
+selenium = "==2.53.6"
+six = "==1.11.0"
+slacker = "==0.9.60"
+tldextract = "==2.1.0"
+"urllib3" = "==1.22"
+"w3lib" = "==1.18.0"
+Automat = "==0.6.0"
+PyDispatcher = "==2.0.5"
+pyOpenSSL = "==17.3.0"
+Scrapy = "==1.5.0"
+service_identity = "==17.0.0"
+Twisted = "==17.5.0"
+"zope.interface" = "==4.4.2"
+
+[dev-packages]
+
+[requires]
+python_version = "2.7"