From c98ec26e5ae54519c4d4d8238a3429e22dc5f1fe Mon Sep 17 00:00:00 2001 From: s-pace Date: Thu, 11 Oct 2018 13:15:41 +0200 Subject: [PATCH 01/17] removing unused impor --- scraper/src/custom_downloader_middleware.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scraper/src/custom_downloader_middleware.py b/scraper/src/custom_downloader_middleware.py index ba6871b9..6175c04e 100644 --- a/scraper/src/custom_downloader_middleware.py +++ b/scraper/src/custom_downloader_middleware.py @@ -5,7 +5,6 @@ import time from scrapy.http import Request, HtmlResponse -from scrapy.exceptions import IgnoreRequest try: from urlparse import urlparse From 8e348c9f74be1667fe9b86cfa47a303c3d1d2d01 Mon Sep 17 00:00:00 2001 From: s-pace Date: Thu, 11 Oct 2018 13:16:14 +0200 Subject: [PATCH 02/17] enhance URL handling --- deployer/src/config_creator.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/deployer/src/config_creator.py b/deployer/src/config_creator.py index a9440034..0c816f13 100644 --- a/deployer/src/config_creator.py +++ b/deployer/src/config_creator.py @@ -3,12 +3,33 @@ import re from . import helpers from . import helpdesk_helper +from urlparse import urlparse def extract_root_from_input(input_string): # We cant parse the url since user might have not enter a proper link domain = re.match(".+?([^\/]\/(?!\/))", input_string) # extracting substring before the first isolated / (not //) - return domain.group() if domain else input_string + + try: + url_parsed = urlparse(input_string); + # Removing unused parameters + url_parsed._replace(params='', query='', fragment='') + path_splited = url_parsed.path.split('/') + path_length = len(path_splited) + + # Path is redirecting to a page + if ('html' in path_splited[path_length - 1]): + url_parsed = url_parsed._replace(path='/'.join(path_splited[: -1])) + # Path note ending by / + elif (path_splited[path_length - 1]): + url_parsed = url_parsed._replace(path='/'.join(path_splited) + '/') + # We are fine + else: + pass + + return url_parsed.geturl() + except ValueError: + return domain.group() if domain else input_string def to_docusaurus_config(config, urls=None): From 311d01187d8b9ca926ae306147efcffce4a2063c Mon Sep 17 00:00:00 2001 From: s-pace Date: Thu, 11 Oct 2018 13:17:47 +0200 Subject: [PATCH 03/17] avoid scraping twice the start_ulrs, removing LICENSE scraping --- deployer/src/config_creator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployer/src/config_creator.py b/deployer/src/config_creator.py index 0c816f13..58019bd2 100644 --- a/deployer/src/config_creator.py +++ b/deployer/src/config_creator.py @@ -82,7 +82,8 @@ def to_pkgdown_config(config, urls=None): config["selectors"]["text"] = ".contents p, .contents li, .usage, .template-article .contents .pre" config["selectors_exclude"] = [".dont-index"] config["custom_settings"] = {"separatorsToIndex": "_"} - # config["stop_urls"] = [start_url + "index.html", "LICENSE-text.html"] + config["scrap_start_urls"] = False + config["stop_urls"] = ["LICENSE-text.html"] return config From 2aaf6dd7fa03636edb66d7baca6a7e9a8d1dae0d Mon Sep 17 00:00:00 2001 From: s-pace Date: Thu, 11 Oct 2018 15:14:39 +0200 Subject: [PATCH 04/17] Enhance contributing --- CONTRIBUTING.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8985d4bd..7f5de178 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,16 +11,35 @@ If you are Algolia employee and want to manage a DocSearch account, you'll need to add the following variables in your `.env` file: ``` +APPLICATION_ID= +API_KEY= +APPLICATION_ID_PROD= +API_KEY_PROD= +HELP_SCOUT_API_KEY= +NB_HITS_MAX= + WEBSITE_USERNAME= WEBSITE_PASSWORD= SLACK_HOOK= SCHEDULER_USERNAME= SCHEDULER_PASSWORD= +MONITORING_API_KEY= +PUBLIC_CONFIG_FOLDER= + /configs/public +PRIVATE_CONFIG_FOLDER= + +BASE_INTERNAL_ENDPOINT= +INTERNAL_API_AUTH= + DEPLOY_KEY= ``` Ping the @docsearch team to get access to those credentials. +## Ready to deploy +Clone the configurations repo in your docsearch-scraper directory. Command to run from the scraper root: +```bash +git clone git@github.com:algolia/docsearch-configs.git configs/public +``` The CLI will then have more commands for you to run. From d500833436f1c248c7f2feae1b274208d8f9aefd Mon Sep 17 00:00:00 2001 From: s-pace Date: Thu, 11 Oct 2018 15:17:55 +0200 Subject: [PATCH 05/17] adding pipfile in order to promote installation with pipenv as requested from algolia/docsearch#449 waiting for pypa/pipenv#2956 --- Pipfile | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 Pipfile diff --git a/Pipfile b/Pipfile new file mode 100644 index 00000000..efba01fd --- /dev/null +++ b/Pipfile @@ -0,0 +1,53 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +algoliasearch = "==1.13.0" +"asn1crypto" = "==0.22.0" +attrs = "==17.2.0" +certifi = "==2017.7.27.1" +cffi = "==1.11.0" +chardet = "==3.0.4" +click = "==6.7" +constantly = "==15.1.0" +cryptography = "==2.0.3" +cssselect = "==1.0.1" +"enum34" = "==1.1.6" +future = "==0.16.0" +hyperlink = "==17.3.1" +idna = "==2.6" +incremental = "==17.5.0" +ipaddress = "==1.0.18" +lxml = "==4.0.0" +ndg-httpsclient = "==0.4.3" +ordereddict = "==1.1" +parsel = "==1.2.0" +"pyasn1" = "==0.3.5" +"pyasn1-modules" = "==0.1.4" +pycparser = "==2.18" +pyperclip = "==1.5.27" +python-dotenv = "==0.7.1" +queuelib = "==1.4.2" +ratelimit = "==1.4.1" +requests = "==2.18.4" +requests-file = "==1.4.2" +selenium = "==2.53.6" +six = "==1.11.0" +slacker = "==0.9.60" +tldextract = "==2.1.0" +"urllib3" = "==1.22" +"w3lib" = "==1.18.0" +Automat = "==0.6.0" +PyDispatcher = "==2.0.5" +pyOpenSSL = "==17.3.0" +Scrapy = "==1.5.0" +service_identity = "==17.0.0" +Twisted = "==17.5.0" +"zope.interface" = "==4.4.2" + +[dev-packages] + +[requires] +python_version = "2.7" From 8813de684a082c2528d088bc2ec7365956233796 Mon Sep 17 00:00:00 2001 From: s-pace Date: Mon, 15 Oct 2018 12:52:36 +0200 Subject: [PATCH 06/17] more elegant use of array and best case scenario --- deployer/src/config_creator.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/deployer/src/config_creator.py b/deployer/src/config_creator.py index 58019bd2..aca670ae 100644 --- a/deployer/src/config_creator.py +++ b/deployer/src/config_creator.py @@ -8,21 +8,20 @@ def extract_root_from_input(input_string): # We cant parse the url since user might have not enter a proper link - domain = re.match(".+?([^\/]\/(?!\/))", input_string) # extracting substring before the first isolated / (not //) + if input_string.endswith('/'): # We assume that the string is already the proper root + return input_string + + domain = re.match(".+?([^\/]\/(?!\/))", input_string) # extracting substring before the first isolated / (not //) try: url_parsed = urlparse(input_string); # Removing unused parameters url_parsed._replace(params='', query='', fragment='') path_splited = url_parsed.path.split('/') - path_length = len(path_splited) # Path is redirecting to a page - if ('html' in path_splited[path_length - 1]): + if ('html' in path_splited[-1]): url_parsed = url_parsed._replace(path='/'.join(path_splited[: -1])) - # Path note ending by / - elif (path_splited[path_length - 1]): - url_parsed = url_parsed._replace(path='/'.join(path_splited) + '/') # We are fine else: pass From 0ecc4e1c93926428ba8bb268b6dcd3f74c94123c Mon Sep 17 00:00:00 2001 From: Pixelastic Date: Tue, 30 Oct 2018 10:58:39 +0100 Subject: [PATCH 07/17] Change link to styling guide --- deployer/src/snippeter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployer/src/snippeter.py b/deployer/src/snippeter.py index 863e688d..6cd79014 100644 --- a/deployer/src/snippeter.py +++ b/deployer/src/snippeter.py @@ -38,7 +38,7 @@ def get_email_for_config(config, analytics_statuses=None): - Add a search input in your page if you don't have any yet. Then update the inputSelector value in JS snippet to a CSS selector that targets your search input field.{{FACETS}} -- Optionally customize the look and feel by following the DocSearch documentation (https://community.algolia.com/docsearch/documentation/) +- Optionally customize the look and feel by following the DocSearch documentation (https://community.algolia.com/docsearch/styling.html) - You can also check your configuration in our github repo (https://github.com/algolia/docsearch-configs/blob/master/configs/{{INDEX_NAME}}.json). {{ANALYTICS}} Please open a pull request if want to leverage your configuration! From 8d65a22c219d03f095fb3eb144988ae245925730 Mon Sep 17 00:00:00 2001 From: s-pace Date: Thu, 11 Oct 2018 15:14:39 +0200 Subject: [PATCH 08/17] Enhance contributing --- CONTRIBUTING.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8985d4bd..7f5de178 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,16 +11,35 @@ If you are Algolia employee and want to manage a DocSearch account, you'll need to add the following variables in your `.env` file: ``` +APPLICATION_ID= +API_KEY= +APPLICATION_ID_PROD= +API_KEY_PROD= +HELP_SCOUT_API_KEY= +NB_HITS_MAX= + WEBSITE_USERNAME= WEBSITE_PASSWORD= SLACK_HOOK= SCHEDULER_USERNAME= SCHEDULER_PASSWORD= +MONITORING_API_KEY= +PUBLIC_CONFIG_FOLDER= + /configs/public +PRIVATE_CONFIG_FOLDER= + +BASE_INTERNAL_ENDPOINT= +INTERNAL_API_AUTH= + DEPLOY_KEY= ``` Ping the @docsearch team to get access to those credentials. +## Ready to deploy +Clone the configurations repo in your docsearch-scraper directory. Command to run from the scraper root: +```bash +git clone git@github.com:algolia/docsearch-configs.git configs/public +``` The CLI will then have more commands for you to run. From 3ee03841838ac47bf6f931b478afdd59c6896010 Mon Sep 17 00:00:00 2001 From: s-pace Date: Thu, 11 Oct 2018 15:17:55 +0200 Subject: [PATCH 09/17] adding pipfile in order to promote installation with pipenv as requested from algolia/docsearch#449 waiting for pypa/pipenv#2956 --- Pipfile | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 Pipfile diff --git a/Pipfile b/Pipfile new file mode 100644 index 00000000..efba01fd --- /dev/null +++ b/Pipfile @@ -0,0 +1,53 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +algoliasearch = "==1.13.0" +"asn1crypto" = "==0.22.0" +attrs = "==17.2.0" +certifi = "==2017.7.27.1" +cffi = "==1.11.0" +chardet = "==3.0.4" +click = "==6.7" +constantly = "==15.1.0" +cryptography = "==2.0.3" +cssselect = "==1.0.1" +"enum34" = "==1.1.6" +future = "==0.16.0" +hyperlink = "==17.3.1" +idna = "==2.6" +incremental = "==17.5.0" +ipaddress = "==1.0.18" +lxml = "==4.0.0" +ndg-httpsclient = "==0.4.3" +ordereddict = "==1.1" +parsel = "==1.2.0" +"pyasn1" = "==0.3.5" +"pyasn1-modules" = "==0.1.4" +pycparser = "==2.18" +pyperclip = "==1.5.27" +python-dotenv = "==0.7.1" +queuelib = "==1.4.2" +ratelimit = "==1.4.1" +requests = "==2.18.4" +requests-file = "==1.4.2" +selenium = "==2.53.6" +six = "==1.11.0" +slacker = "==0.9.60" +tldextract = "==2.1.0" +"urllib3" = "==1.22" +"w3lib" = "==1.18.0" +Automat = "==0.6.0" +PyDispatcher = "==2.0.5" +pyOpenSSL = "==17.3.0" +Scrapy = "==1.5.0" +service_identity = "==17.0.0" +Twisted = "==17.5.0" +"zope.interface" = "==4.4.2" + +[dev-packages] + +[requires] +python_version = "2.7" From 321b1e05e2a35e36601c28fa81cc11af44a36acf Mon Sep 17 00:00:00 2001 From: s-pace Date: Fri, 2 Nov 2018 18:14:59 +0100 Subject: [PATCH 10/17] pipenv --python 2.7 install --- Pipfile.lock | 495 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 495 insertions(+) create mode 100644 Pipfile.lock diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 00000000..e60753b9 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,495 @@ +{ + "_meta": { + "hash": { + "sha256": "ea63d2fa61c0ae7f744909e76b3a0f2095d187c28a3523e2ac4f199c7361f717" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "2.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "algoliasearch": { + "hashes": [ + "sha256:434eb2f04060301cfb12fad078fa85f6f1b70ecdfdbebd093e92380a957f16cb", + "sha256:fcdfd8605c6ace8d5a225a5eb679e837548d85a8fa234a57296eba69ab259797" + ], + "index": "pypi", + "version": "==1.13.0" + }, + "asn1crypto": { + "hashes": [ + "sha256:cbbadd640d3165ab24b06ef25d1dca09a3441611ac15f6a6b452474fdf0aed1a", + "sha256:d232509fefcfcdb9a331f37e9c9dc20441019ad927c7d2176cf18ed5da0ba097" + ], + "index": "pypi", + "version": "==0.22.0" + }, + "attrs": { + "hashes": [ + "sha256:5d4d1b99f94d69338f485984127e4473b3ab9e20f43821b0e546cc3b2302fd11", + "sha256:a7e0d9183f6457de12df7ba6a81f6569c7d6b25f67ad509b5ad52e8545970a2f" + ], + "index": "pypi", + "version": "==17.2.0" + }, + "automat": { + "hashes": [ + "sha256:2140297df155f7990f6f4c73b2ab0583bd8150db9ed2a1b48122abe66e9908c1", + "sha256:3c1fd04ecf08ac87b4dd3feae409542e9bf7827257097b2b6ed5692f69d6f6a8" + ], + "index": "pypi", + "version": "==0.6.0" + }, + "certifi": { + "hashes": [ + "sha256:40523d2efb60523e113b44602298f0960e900388cf3bb6043f645cf57ea9e3f5", + "sha256:54a07c09c586b0e4c619f02a5e94e36619da8e2b053e20f594348c0611803704" + ], + "index": "pypi", + "version": "==2017.7.27.1" + }, + "cffi": { + "hashes": [ + "sha256:040e0b5462c65a39b7112ad9e951f3f57b487d0db053ea510dcf8fc53931ab4b", + "sha256:2217d22cb001386947693bd93ca4370224989468daf2939f3bf717956fb0a119", + "sha256:42b38cd48e55b2a4c5f038f80137c2cb4393fd4e4d1c3dbeeb7aea9475b68ea1", + "sha256:45233398f5e7114e719d0b83ffd0642af7fbf05136195f500fb6b72133c8e998", + "sha256:47853d133f0de32697b55e0c2ee377d88e3143bc8645a3b93d38be2f5387b0a2", + "sha256:48d6589fda4ec0ccb7d973807890a733381f828d29167177eb7d97892255a769", + "sha256:4d2e388ce51e6a4efae8d4533ca15503ea957c41233bad165f8ccec354edca67", + "sha256:5152bbde6ee514a11e2cd1b76a7ccab5af1043356ec3df9efe0112d97b20dae3", + "sha256:56a8c39e6e7225474b36c16a03240fcbbfc5c8cb016645793e0dec2d2f9bdf67", + "sha256:59fc30a6f7331fe69cfc67ef151fd03a33491d25430ae0c4597e69c4235fa87e", + "sha256:5c20f34a6858cd8c59eac23f80bc80bb5fd294bd9b22b264255a3eec0ab459f5", + "sha256:5f4ff33371c6969b39b293d9771ee91e81d26f9129be093ca1b7be357fcefd15", + "sha256:6153b26756de2f34f80ca4e8b1e79011ad62bb670539dc3df71e62909a112fdf", + "sha256:6f1a2f85510c675b068e29625ff534f02567f24839cbd7d61cda6b7df62da2b1", + "sha256:765cf591a2f790b012075c7f54344a02b622a8561708ec27b3042c8c1699a465", + "sha256:7a55f044dd78fdca379d455550c581b9455af022b8f8b5d97f585a25f4da63f6", + "sha256:8a5632592b2752375b6d9e81da022896ece235713129fcc33703b3e10c639ef9", + "sha256:929d206ecfefadcec0b0e13956131eec7442829d240da020d67274fefb9e61a4", + "sha256:a76b89adecd46138aafdbd348563df1deb714962b5b3659ee1b6b0c5cf6ae0e5", + "sha256:b45efc8d81640a8e9122c1ba1817da2a998ca0168c40de7bbfc47b960771ca94", + "sha256:bd93a0df4dc72893ba604c05d487ea4d47a799e54172b89e414953685b0b08fb", + "sha256:c091a1ac521dac2770632f499c124762ddf978d23ced0472f19a8830e006eab8", + "sha256:d361e8c504bae02f665fad8d5ba4b74a6c1fab7862eec39cddcc21e4479fa275", + "sha256:d8930a719781987a876d6de8cf33ce937191c383158daafcb9db804bf3421adf", + "sha256:dd9577197fff563e4784ef4ace7da28c7da96a81895b69c008cdb82f1e82853c", + "sha256:ec1112466a924ba69df5a90f31d26d323e3b91ce21f94b10fa0b77f44a07df48", + "sha256:eddf6bd95b100f34c56a226847be92bde416d028213e87fd700bc593cbf3e9da", + "sha256:f14c53dc624529eef175c993ceb886f37ecece09825bf47bf3230aacbc3b7acc", + "sha256:f2ffdfd930bc4bcd4900def32fbb2a16a91656d9caafff573a25934327140431", + "sha256:f6202eab19fd2c44d7d7af2bb8d286cf18b7b3d67b0e5bf1f2f113c99303ff87", + "sha256:fd3c49d7d22ba75382e19c4877e1d4dca232ef48137442f8582c2259b0cdf612" + ], + "index": "pypi", + "version": "==1.11.0" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "index": "pypi", + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", + "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" + ], + "index": "pypi", + "version": "==6.7" + }, + "constantly": { + "hashes": [ + "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35", + "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d" + ], + "index": "pypi", + "version": "==15.1.0" + }, + "cryptography": { + "hashes": [ + "sha256:0f3dcdd06c5491cad2ed283ebc802805e3ac77ffe4789ad1b44ff0f0f891f59b", + "sha256:15d655434cf6f350462c6c97f24cbf42ee0e6270886a7937eaaa52fd09e474ce", + "sha256:1fb7dbf630daba53494da88de1e87429f2740e6ffdc92bbdcb385ddb9ff67b32", + "sha256:27b7b005debcf4c0562299d5ceeaea151120eaad7bd84ef91fb22c204a958f03", + "sha256:2f0b681e89530750e4c0adf98701cafd0384ac160ceafe39c9d10be6034d2cc7", + "sha256:317127295b14f5c943ed789ab60ade0d7d39041f8975d58a214af19dd1148705", + "sha256:369e4669cac8296e320f86d0a3c9f489b7593d2a144e7b0504b884bed56fd5d7", + "sha256:4ef205496c518aae7e699731dfff4b630c046ea9dbbe34e8ec8ce77bb399c1c7", + "sha256:4ffc706fbe4c12146050d432250896f2fcad8b77c6fd89d3c3bd456884dbf421", + "sha256:5544d9433f9c25d55d609e2b014d47c390288bb98ae84e35b8cf93e9f6a6c832", + "sha256:55cc98c93323efcf6f3300b6e9a7d435146a9d451ffd27aa14074f576f275786", + "sha256:59d2b285fb24d30deaf4143c2d048fcefdd7e560f30cd1745da881705ca965c6", + "sha256:5a266ff306683cd82b2f19203f469b7580aee115f1818d2b2d1e12f1ee7fd12b", + "sha256:5c9e46b219e2b65ab9192e10407d7698589794407f15f11f39abbf3b04d16890", + "sha256:795af466e52daf404d783877157b106bc79e0cffa55f8d6360fa9f9fd03148bd", + "sha256:8067fce92ec98cb52fabe2e10205ff27dba5755d9ffc1f521be9c504f9a6d5fc", + "sha256:886759453c5b93b1e8953e92abee65ee09fdf78bf655c95d0a206d6387adbfed", + "sha256:8bbff1de3002db5c7146b88fea392526260becc153adc8810103fc750c3eb333", + "sha256:a83318144a0037f264891ddfcbae9b75303949bb0efaece9713df5e04388b7ef", + "sha256:a972eb37b6409a2d67ddd956e6bf74f2e9315105083d1fdaa8bbe6eb520dc367", + "sha256:b17e4244212c9ec6526b6ad47534a12e0378387ad86228141053c112e759769d", + "sha256:b25d97585f1d2b2f16560737aba6257a10ca18aa3a2a70407c05fe8b64b375f9", + "sha256:bb715b0d8d5c9111e9656eabe58ee7b95842a797c04608aef7f52439ab73d5f3", + "sha256:bc86d3de2c7d63f3c73dff24fa43c55c00c7b8ea8102ed638c903c939b85e6a6", + "sha256:d04bb2425086c3fe86f7bc48915290b13e798497839fbb18ab7f6dffcf98cc3a", + "sha256:d32dad8448ea88fe8eb02dc943d46db658cf9e618386761e498802c79e97c937", + "sha256:d44db8e4435a10dcbf22fbc03072d18b31e6c1100084a2010239efd359f2f162", + "sha256:d5341dac22e8ab080e637c632d1b1e92acaa8e66a02d139c0c795c1797531384", + "sha256:fb7dfc4b3a79ca4b9233803b80617fd302374e57771c9a28e32ee6c1296cdea3", + "sha256:fe6d56a4ff08875d70cc9b336fcd6e938db5808cb47cbc33c6927affb659aded" + ], + "index": "pypi", + "version": "==2.0.3" + }, + "cssselect": { + "hashes": [ + "sha256:4f5f799a1d3182b04814007e9e7fc6c362f4489c7420d6b348cc901ece07ced9", + "sha256:73db1c054b9348409e2862fc6c0dde5c4e4fbe4da64c5c5a9e05fbea45744077" + ], + "index": "pypi", + "version": "==1.0.1" + }, + "enum34": { + "hashes": [ + "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", + "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", + "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", + "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" + ], + "index": "pypi", + "version": "==1.1.6" + }, + "future": { + "hashes": [ + "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + ], + "index": "pypi", + "version": "==0.16.0" + }, + "hyperlink": { + "hashes": [ + "sha256:1ec8e11fb4f5b330f25864bf8cfd3133dff1a3637dfd14fa441297df15fc7cf9", + "sha256:bc4ffdbde9bdad204d507bd8f554f16bba82dd356f6130cb16f41422909c33bc" + ], + "index": "pypi", + "version": "==17.3.1" + }, + "idna": { + "hashes": [ + "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f", + "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4" + ], + "index": "pypi", + "version": "==2.6" + }, + "incremental": { + "hashes": [ + "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f", + "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3" + ], + "index": "pypi", + "version": "==17.5.0" + }, + "ipaddress": { + "hashes": [ + "sha256:5d8534c8e185f2d8a1fda1ef73f2c8f4b23264e8e30063feeb9511d492a413e1", + "sha256:d34cf15d95ce9a734560f7400a8bd2ac2606f378e2a1d0eadbf1c98707e7c74a" + ], + "index": "pypi", + "version": "==1.0.18" + }, + "lxml": { + "hashes": [ + "sha256:0b8f3d6e669ea26849a6184f04c7802dbef6fd418a8b90e6c026e237db07af31", + "sha256:10c86b2248043f4428be33ed10202764b02b281eaa4550f16f0fbbc6ccaae9ac", + "sha256:1f81074e77c25f9b787fa3854f400ca924d3d798cb7ae910c0e7920be7138c90", + "sha256:3593f49858fc6229cd93326be06b099ae477fd65d8f4a981320a6d0bb7fc7a5a", + "sha256:48ab0e79175fd16f9478edc679ee14c79524c64b26f665f92cbecff81312d04d", + "sha256:52e18dd86f153c4383bb4c4ef62f81f9b7e44809d068848a5a183b2285496faf", + "sha256:550a51dee73c14e5863bdbbbe5836b2b8092a3f92631b5a908b9c41e72f123a5", + "sha256:567b76f291a8d02aa8b4d3f8295ae749ac4d532570d8a8c7176f0556c7d95891", + "sha256:61825daaf2d80dc3da7635ee108720b0739962db008343822753bbf343cbfd14", + "sha256:7030f44b758e930fd09ade87d770f5a231a19a8c561a3acc54e5122b5ec09e29", + "sha256:824664493a012f5b70396e841a4b4049bdaf445a70307e60f82fe35619f72cc7", + "sha256:88d137e440b5de35df2e0616da8e28a88d0119abdaa84520ad1ba815ee9da732", + "sha256:8996df6b0f205b00b89bbd04d88f1fa1e04139a025fd291aa4ddd05dc86836f4", + "sha256:9581b438e5d0d0a6fa3937fac2abffd95380bd513bcd39f6952bfcf20cf0b9a7", + "sha256:99b7eabfb46663ed5918eca4ed12420613ba24196964a741ccd962d09296c0b2", + "sha256:9f399c37b8e61c3989ef12ecf0abd9c10a5075f0fc9ad1ecd67ce6f9c72a7211", + "sha256:a211288459c9115ddb255ff88e8ac12dc2063e70bddc15e3c65136477a358bb5", + "sha256:a8ad0adeedbbb7b85916214fcd4f5d02829d0e7b3c32abc298789218b6c3d699", + "sha256:b2ecb3fd5470b740dfc21b064bbc1337be4b7b805994a868488145d36f35f517", + "sha256:b7f6ef610680709be11cb7662e46e254bc561dafe0de3b4848be2cf3924bd300", + "sha256:c446fde3284c363cd2085ad1ce5a07c18f15f6766d72684622bc14b0a9ddfd29", + "sha256:c4e02657e629f02ab8712471d77d6896c2cf6f09f8ffa6a0f23b1b1ef0318474", + "sha256:d4507916c408feec2ea8cee3f0d1380e49ea431f6e07b0dd927388bd6e92d6eb", + "sha256:d78c0a114cf127a41a526aef99aef539c0b2537e57f04a2cc7a49e2c94a44ab8", + "sha256:d9ec728caddb161405e7c33ed9d176e96309893481370163bbf4b00e43008795", + "sha256:e37eda3d05519918403084b43eb7324df21a0daf45c8ae8172a860710dd0fa78", + "sha256:e908d685800626f10cd6ae01a013fc42094be167fb2a683eb920dfddfaa0ee76", + "sha256:f7bc9f702500e205b1560d620f14015fec76dcd6f9e889a946a2ddcc3c344fd0" + ], + "index": "pypi", + "version": "==4.0.0" + }, + "ndg-httpsclient": { + "hashes": [ + "sha256:28d62f2fc93dcd20793ddf2ce9ec1ab9f3458a11a784105a6d67f218ca53de63", + "sha256:5b1e1b1991990487e28e5b95e062c9d06f8f00e3cd79eaddc448ce06c5e3b776", + "sha256:7bfd8c5cfcbc241a93ca6a4e45f952650f5c7ecf7c49b1dbcf5f4d390240be0b" + ], + "index": "pypi", + "version": "==0.4.3" + }, + "ordereddict": { + "hashes": [ + "sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f" + ], + "index": "pypi", + "version": "==1.1" + }, + "parsel": { + "hashes": [ + "sha256:1c5a31cb3f0fdca3d19f2ef966d595df69181c2d81b6c56582732c819e2f4e26", + "sha256:29eb4bd74e22e41138ea8ed96ce8e477b8116f97a13a991e39cb150fdde7eabd" + ], + "index": "pypi", + "version": "==1.2.0" + }, + "pyasn1": { + "hashes": [ + "sha256:90a6dfe991beb1f6240bc19e9fda02f513517523e3b96474a0887a94022dbad0", + "sha256:c7a1186416bcdaf19b56f5b2506200306d7d4cf222d3d7c2d4cb600ccacdaf1b" + ], + "index": "pypi", + "version": "==0.3.5" + }, + "pyasn1-modules": { + "hashes": [ + "sha256:529d5d509562c22877a53771c5b394d8102c98cd62f33bb5af23b475879ea6d5", + "sha256:b07c17bdb34d6f64aafea6269f2e8fb306a57473f0f38d9a6ca389d6ab30ac4a" + ], + "index": "pypi", + "version": "==0.1.4" + }, + "pycparser": { + "hashes": [ + "sha256:99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226" + ], + "index": "pypi", + "version": "==2.18" + }, + "pydispatcher": { + "hashes": [ + "sha256:5570069e1b1769af1fe481de6dd1d3a388492acddd2cdad7a3bde145615d5caf", + "sha256:5be4a8be12805ef7d712dd9a93284fb8bc53f309867e573f653a72e5fd10e433" + ], + "index": "pypi", + "version": "==2.0.5" + }, + "pyopenssl": { + "hashes": [ + "sha256:29630b9064a82e04d8242ea01d7c93d70ec320f5e3ed48e95fcabc6b1d0f6c76", + "sha256:aade9985b93eaec51b0c0a2a60d14bb8dcff1ff8e36fe542e3c22812ec07315e" + ], + "index": "pypi", + "version": "==17.3.0" + }, + "pyperclip": { + "hashes": [ + "sha256:a3cb6df5d8f1557ca8fc514d94fabf50dc5a97042c90e5ba4f3611864fed3fc5" + ], + "index": "pypi", + "version": "==1.5.27" + }, + "python-dotenv": { + "hashes": [ + "sha256:45e927c34204c90f5faa35ea8709b894f6b1a7712d77eb50940601068040993b", + "sha256:dc7940052cfe170e881aea40feb4ea7776e6a97170ed038fd2ee7e26e47585f2" + ], + "index": "pypi", + "version": "==0.7.1" + }, + "queuelib": { + "hashes": [ + "sha256:a6829918157ed433fafa87b0bb1e93e3e63c885270166db5884a02c34c86f914", + "sha256:c97f2120e070e2b8a5e8a60afe10bd7b3ae70c0994d6bb9c4d802b0448241339" + ], + "index": "pypi", + "version": "==1.4.2" + }, + "ratelimit": { + "hashes": [ + "sha256:d001f4dacf4880f64dafb5bc6ef0cc882aa94f291c623bbcf080e0a882e62351" + ], + "index": "pypi", + "version": "==1.4.1" + }, + "requests": { + "hashes": [ + "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", + "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" + ], + "index": "pypi", + "version": "==2.18.4" + }, + "requests-file": { + "hashes": [ + "sha256:f518e7cfe048e053fd1019dfb891b4c55b871c56c5a31693d733240c80b8f191" + ], + "index": "pypi", + "version": "==1.4.2" + }, + "scrapy": { + "hashes": [ + "sha256:08d86737c560dcc1c4b73ac0ac5bd8d14b3e2265c1f7b195f0b73ab13741fe03", + "sha256:31a0bf05d43198afaf3acfb9b4fb0c09c1d7d7ff641e58c66e36117f26c4b755" + ], + "index": "pypi", + "version": "==1.5.0" + }, + "selenium": { + "hashes": [ + "sha256:5071f43daa2e698d60d5633ab0a6630cc68a852b360be99144f1c4c1ace2746c", + "sha256:f507181f13768d73b98dd9647a466ea5758ef5c7f07b62a285d2bd8de9b27016" + ], + "index": "pypi", + "version": "==2.53.6" + }, + "service-identity": { + "hashes": [ + "sha256:0e76f3c042cc0f5c7e6da002cf646f59dc4023962d1d1166343ce53bdad39e17", + "sha256:4001fbb3da19e0df22c47a06d29681a398473af4aa9d745eca525b3b2c2302ab" + ], + "index": "pypi", + "version": "==17.0.0" + }, + "six": { + "hashes": [ + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + ], + "index": "pypi", + "version": "==1.11.0" + }, + "slacker": { + "hashes": [ + "sha256:5d54950a1956c6faff3c82cd116aed4436cf0e322a72747d77b9189d5ec7f8cb" + ], + "index": "pypi", + "version": "==0.9.60" + }, + "tldextract": { + "hashes": [ + "sha256:6a705710f3a1d5ce8a084c888db7bad7aff1dcdf27962f7c7f7058cc42349212", + "sha256:796d86aa22c382770b28db45547b298cbf855dcdcd8e1c7264419d96c0f2381a" + ], + "index": "pypi", + "version": "==2.1.0" + }, + "twisted": { + "hashes": [ + "sha256:60cfb24a20a14d8113bbbdc15601da895f6c025189cb660b6dd1e73623675419", + "sha256:f198a494f0df2482f7c5f99d7f3eef33d22763ffc76641b36fec476b878002ea" + ], + "index": "pypi", + "version": "==17.5.0" + }, + "urllib3": { + "hashes": [ + "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", + "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" + ], + "index": "pypi", + "version": "==1.22" + }, + "w3lib": { + "hashes": [ + "sha256:7b661935805b7d39afe90bb32dec8e4d20b377e74c66597eb1ddfad10071938e", + "sha256:c48731d5d73cde86f9c3c2bd6898d165f670120427353a7c8f9d6c685561d3c4" + ], + "index": "pypi", + "version": "==1.18.0" + }, + "zope-interface": { + "hashes": [ + "sha256:0b84b11b828770f9d711be99d037bb076ed0291a343fd910b683a9bd7866e0f2", + "sha256:0d1e1f3a816b5b4774c0a7cf46adf399d298f44c4de71ecab6ea83b9b71b9bc7", + "sha256:0e9fa25e4786d02e88a17f0b1152ab3e901ab47a44b05a6c711e253836d4698c", + "sha256:0effab9d51933e35ccd01f4c8c6d72f8040df78deacb2ec9176a7519722ce9ba", + "sha256:13119a6ab5cd1d9b28b85ac9c7e1dadd4e0b743016a5b77a374ce51b3d26f339", + "sha256:17873e481cd2dce1db4fc15bbcb59400c100771354bddad8302fadb7dd2a0f0e", + "sha256:21c506b586ab17e7d353e5597683112285b0889330c9bd4f374e4f8b387d3348", + "sha256:2bcd66cc16dccf8b1cca37abe8ce947d38b54f78c244fd45102fc00997c7446a", + "sha256:3c87138213b2a3896adcbf4fcc3dd916c68cef405cadcb9327d0f08949e10194", + "sha256:40026b28cbf065ae54dcfdb1f95f14d6d07c4f38a4293a3bb663f2bee620fd4e", + "sha256:5c14d72a86df23bc473efa8567a2f04fc94d1ba84a8ba651e3a9991e8b20a15a", + "sha256:5e0c85069a508835b192d5fd304aef4a7b23a325f2cd58e8953ae1c1eb1a8570", + "sha256:6c9e61ac24ed84d9c6a20155d979bd08f278c76450d7ab65a107057780457aac", + "sha256:74eba4f1d2100424d51956103836d7c07bc8e5bf17fcf34a59b784e1fcedeaf2", + "sha256:74f2a8c466124f852cb78bbb6bf8306340af1885e77870abc5418cbfeb54fece", + "sha256:7cc6f03a905094983f92cd6a97244de05e9293f7b09e110ce4c7c81de8a26a3a", + "sha256:8aa8f86fb2c7a7cab830fc871a2bd19af18dabb0e940ec9adb658136eb12ebae", + "sha256:8adee390aaeeae7d5858f023e552bdb631ff4f129900ea1e8bb62c3722af12f5", + "sha256:9a567f17f3f41e955bc87cfa39cbd5713ba3077dfd8a06a360605ec45407e357", + "sha256:aeeff785ba7cd5cad4b2cfd2930bf5c2431e0fdfdf7ffc0b3af44827f1007611", + "sha256:b2467ebf96561f022265cf0e1920d19ab1178deb3f0bb9fbd81bd90e995251c1", + "sha256:c55003bf2ab1046dbc0e79a4a9ac7af2f5800ed6ec4cb8e35fc3c99a21f380a8", + "sha256:dac9c7d45f8ab94f6f1a0dd167398a633924c85238cc8d76a6a04e6eac04def7", + "sha256:f7cbfd9698c16cb2ed164e7b9e4b3a24217e51fccfd95c3fd3dcca9bc1e7c9e9", + "sha256:fae7b143aea7248cbbed9ed3977e8198fa82805d1643556b03d23df28ffdf99b", + "sha256:fbe4140cc7743c541c302a0bf31dca432b0c0a914053809a9338237d54c3b6ca", + "sha256:fd56bbded257c0da54e98dce7a030e76cc018b8c3eaeb8ce8f35487fb1c859e3" + ], + "index": "pypi", + "version": "==4.4.2" + }, + "zope.interface": { + "hashes": [ + "sha256:086707e0f413ff8800d9c4bc26e174f7ee4c9c8b0302fbad68d083071822316c", + "sha256:1157b1ec2a1f5bf45668421e3955c60c610e31913cc695b407a574efdbae1f7b", + "sha256:11ebddf765bff3bbe8dbce10c86884d87f90ed66ee410a7e6c392086e2c63d02", + "sha256:14b242d53f6f35c2d07aa2c0e13ccb710392bcd203e1b82a1828d216f6f6b11f", + "sha256:1b3d0dcabc7c90b470e59e38a9acaa361be43b3a6ea644c0063951964717f0e5", + "sha256:20a12ab46a7e72b89ce0671e7d7a6c3c1ca2c2766ac98112f78c5bddaa6e4375", + "sha256:298f82c0ab1b182bd1f34f347ea97dde0fffb9ecf850ecf7f8904b8442a07487", + "sha256:2f6175722da6f23dbfc76c26c241b67b020e1e83ec7fe93c9e5d3dd18667ada2", + "sha256:3b877de633a0f6d81b600624ff9137312d8b1d0f517064dfc39999352ab659f0", + "sha256:4265681e77f5ac5bac0905812b828c9fe1ce80c6f3e3f8574acfb5643aeabc5b", + "sha256:550695c4e7313555549aa1cdb978dc9413d61307531f123558e438871a883d63", + "sha256:5f4d42baed3a14c290a078e2696c5f565501abde1b2f3f1a1c0a94fbf6fbcc39", + "sha256:62dd71dbed8cc6a18379700701d959307823b3b2451bdc018594c48956ace745", + "sha256:7040547e5b882349c0a2cc9b50674b1745db551f330746af434aad4f09fba2cc", + "sha256:7e099fde2cce8b29434684f82977db4e24f0efa8b0508179fce1602d103296a2", + "sha256:7e5c9a5012b2b33e87980cee7d1c82412b2ebabcb5862d53413ba1a2cfde23aa", + "sha256:81295629128f929e73be4ccfdd943a0906e5fe3cdb0d43ff1e5144d16fbb52b1", + "sha256:95cc574b0b83b85be9917d37cd2fad0ce5a0d21b024e1a5804d044aabea636fc", + "sha256:968d5c5702da15c5bf8e4a6e4b67a4d92164e334e9c0b6acf080106678230b98", + "sha256:9e998ba87df77a85c7bed53240a7257afe51a07ee6bc3445a0bf841886da0b97", + "sha256:a0c39e2535a7e9c195af956610dba5a1073071d2d85e9d2e5d789463f63e52ab", + "sha256:a15e75d284178afe529a536b0e8b28b7e107ef39626a7809b4ee64ff3abc9127", + "sha256:a6a6ff82f5f9b9702478035d8f6fb6903885653bff7ec3a1e011edc9b1a7168d", + "sha256:b639f72b95389620c1f881d94739c614d385406ab1d6926a9ffe1c8abbea23fe", + "sha256:bad44274b151d46619a7567010f7cde23a908c6faa84b97598fd2f474a0c6891", + "sha256:bbcef00d09a30948756c5968863316c949d9cedbc7aabac5e8f0ffbdb632e5f1", + "sha256:d788a3999014ddf416f2dc454efa4a5dbeda657c6aba031cf363741273804c6b", + "sha256:eed88ae03e1ef3a75a0e96a55a99d7937ed03e53d0cffc2451c208db445a2966", + "sha256:f99451f3a579e73b5dd58b1b08d1179791d49084371d9a47baad3b22417f0317" + ], + "version": "==4.6.0" + } + }, + "develop": {} +} From 3051f05d15a041315d9a8cc3946ae7622e2d601b Mon Sep 17 00:00:00 2001 From: s-pace Date: Fri, 2 Nov 2018 18:20:31 +0100 Subject: [PATCH 11/17] give instructions regarding how to use our scraper without docker closes algolia/docsearch#449 --- CONTRIBUTING.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f5de178..79c33dfd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,16 @@ Hello you, + +## Install the scrapper without Docker + +We do recommend the use of [pipenv][1] to install the whole python environment + +- [Install pipenv][2] +- `pipenv --python 2.7 install` + +You should be ready to go. + + ## Run the tests To run the full test suite, run `./docsearch test`. Note that it requires that @@ -42,4 +53,6 @@ git clone git@github.com:algolia/docsearch-configs.git configs/public ``` The CLI will then have more commands for you to run. +[1]: https://github.com/pypa/pipenv +[2]: https://pipenv.readthedocs.io/en/latest/install/#installing-pipenv From ead2d4dbb46132b1457aa4645ee4dce3322419de Mon Sep 17 00:00:00 2001 From: s-pace Date: Fri, 2 Nov 2018 18:23:11 +0100 Subject: [PATCH 12/17] format .md --- CONTRIBUTING.md | 8 ++++---- README.md | 1 - cli/oh-my-zsh/README.md | 4 ++-- deployer/README.md | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 79c33dfd..cbffbbb0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,5 @@ Hello you, - ## Install the scrapper without Docker We do recommend the use of [pipenv][1] to install the whole python environment @@ -10,7 +9,6 @@ We do recommend the use of [pipenv][1] to install the whole python environment You should be ready to go. - ## Run the tests To run the full test suite, run `./docsearch test`. Note that it requires that @@ -47,12 +45,14 @@ DEPLOY_KEY= Ping the @docsearch team to get access to those credentials. ## Ready to deploy + Clone the configurations repo in your docsearch-scraper directory. Command to run from the scraper root: + ```bash git clone git@github.com:algolia/docsearch-configs.git configs/public ``` -The CLI will then have more commands for you to run. + +The CLI will then have more commands for you to run. [1]: https://github.com/pypa/pipenv [2]: https://pipenv.readthedocs.io/en/latest/install/#installing-pipenv - diff --git a/README.md b/README.md index e64b61c0..153b315f 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,6 @@ DocSearch is made of 3 repositories: from your documentation. The code is open-source and you can run it from a Docker image - [1]: https://community.algolia.com/docsearch/ [2]: https://community.algolia.com/docsearch/run-your-own.html [3]: https://github.com/algolia/docsearch diff --git a/cli/oh-my-zsh/README.md b/cli/oh-my-zsh/README.md index 84d1662c..6941b7ca 100644 --- a/cli/oh-my-zsh/README.md +++ b/cli/oh-my-zsh/README.md @@ -1,4 +1,4 @@ ## How to install -- copy the ```docsearch``` folder in ```~/.oh-my-zsh/custom/plugins/``` -- enable the plugin in your ```zshrc``` by modify the plugins variable: ``plugins=([...], docsearch) \ No newline at end of file +- copy the `docsearch` folder in `~/.oh-my-zsh/custom/plugins/` +- enable the plugin in your `zshrc` by modify the plugins variable: ``plugins=([...], docsearch) diff --git a/deployer/README.md b/deployer/README.md index c3742f08..7989ab63 100644 --- a/deployer/README.md +++ b/deployer/README.md @@ -6,4 +6,4 @@ API_KEY="" \ WEBSITE_USERNAME="" \ WEBSITE_PASSWORD="" \ ./deploy -``` \ No newline at end of file +``` From c60588a3ec8843d7ba71620681bcc831de6d0813 Mon Sep 17 00:00:00 2001 From: s-pace Date: Thu, 11 Oct 2018 15:14:39 +0200 Subject: [PATCH 13/17] Enhance contributing --- CONTRIBUTING.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8985d4bd..7f5de178 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,16 +11,35 @@ If you are Algolia employee and want to manage a DocSearch account, you'll need to add the following variables in your `.env` file: ``` +APPLICATION_ID= +API_KEY= +APPLICATION_ID_PROD= +API_KEY_PROD= +HELP_SCOUT_API_KEY= +NB_HITS_MAX= + WEBSITE_USERNAME= WEBSITE_PASSWORD= SLACK_HOOK= SCHEDULER_USERNAME= SCHEDULER_PASSWORD= +MONITORING_API_KEY= +PUBLIC_CONFIG_FOLDER= + /configs/public +PRIVATE_CONFIG_FOLDER= + +BASE_INTERNAL_ENDPOINT= +INTERNAL_API_AUTH= + DEPLOY_KEY= ``` Ping the @docsearch team to get access to those credentials. +## Ready to deploy +Clone the configurations repo in your docsearch-scraper directory. Command to run from the scraper root: +```bash +git clone git@github.com:algolia/docsearch-configs.git configs/public +``` The CLI will then have more commands for you to run. From 3a0bb756bae5c58f90a441d617e7ecd2655c6092 Mon Sep 17 00:00:00 2001 From: s-pace Date: Thu, 11 Oct 2018 15:17:55 +0200 Subject: [PATCH 14/17] adding pipfile in order to promote installation with pipenv as requested from algolia/docsearch#449 waiting for pypa/pipenv#2956 --- Pipfile | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 Pipfile diff --git a/Pipfile b/Pipfile new file mode 100644 index 00000000..efba01fd --- /dev/null +++ b/Pipfile @@ -0,0 +1,53 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +algoliasearch = "==1.13.0" +"asn1crypto" = "==0.22.0" +attrs = "==17.2.0" +certifi = "==2017.7.27.1" +cffi = "==1.11.0" +chardet = "==3.0.4" +click = "==6.7" +constantly = "==15.1.0" +cryptography = "==2.0.3" +cssselect = "==1.0.1" +"enum34" = "==1.1.6" +future = "==0.16.0" +hyperlink = "==17.3.1" +idna = "==2.6" +incremental = "==17.5.0" +ipaddress = "==1.0.18" +lxml = "==4.0.0" +ndg-httpsclient = "==0.4.3" +ordereddict = "==1.1" +parsel = "==1.2.0" +"pyasn1" = "==0.3.5" +"pyasn1-modules" = "==0.1.4" +pycparser = "==2.18" +pyperclip = "==1.5.27" +python-dotenv = "==0.7.1" +queuelib = "==1.4.2" +ratelimit = "==1.4.1" +requests = "==2.18.4" +requests-file = "==1.4.2" +selenium = "==2.53.6" +six = "==1.11.0" +slacker = "==0.9.60" +tldextract = "==2.1.0" +"urllib3" = "==1.22" +"w3lib" = "==1.18.0" +Automat = "==0.6.0" +PyDispatcher = "==2.0.5" +pyOpenSSL = "==17.3.0" +Scrapy = "==1.5.0" +service_identity = "==17.0.0" +Twisted = "==17.5.0" +"zope.interface" = "==4.4.2" + +[dev-packages] + +[requires] +python_version = "2.7" From 8bcb25d437104fa7888c669adb51f1ae5c998a9c Mon Sep 17 00:00:00 2001 From: s-pace Date: Fri, 2 Nov 2018 18:14:59 +0100 Subject: [PATCH 15/17] pipenv --python 2.7 install --- Pipfile.lock | 495 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 495 insertions(+) create mode 100644 Pipfile.lock diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 00000000..e60753b9 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,495 @@ +{ + "_meta": { + "hash": { + "sha256": "ea63d2fa61c0ae7f744909e76b3a0f2095d187c28a3523e2ac4f199c7361f717" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "2.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "algoliasearch": { + "hashes": [ + "sha256:434eb2f04060301cfb12fad078fa85f6f1b70ecdfdbebd093e92380a957f16cb", + "sha256:fcdfd8605c6ace8d5a225a5eb679e837548d85a8fa234a57296eba69ab259797" + ], + "index": "pypi", + "version": "==1.13.0" + }, + "asn1crypto": { + "hashes": [ + "sha256:cbbadd640d3165ab24b06ef25d1dca09a3441611ac15f6a6b452474fdf0aed1a", + "sha256:d232509fefcfcdb9a331f37e9c9dc20441019ad927c7d2176cf18ed5da0ba097" + ], + "index": "pypi", + "version": "==0.22.0" + }, + "attrs": { + "hashes": [ + "sha256:5d4d1b99f94d69338f485984127e4473b3ab9e20f43821b0e546cc3b2302fd11", + "sha256:a7e0d9183f6457de12df7ba6a81f6569c7d6b25f67ad509b5ad52e8545970a2f" + ], + "index": "pypi", + "version": "==17.2.0" + }, + "automat": { + "hashes": [ + "sha256:2140297df155f7990f6f4c73b2ab0583bd8150db9ed2a1b48122abe66e9908c1", + "sha256:3c1fd04ecf08ac87b4dd3feae409542e9bf7827257097b2b6ed5692f69d6f6a8" + ], + "index": "pypi", + "version": "==0.6.0" + }, + "certifi": { + "hashes": [ + "sha256:40523d2efb60523e113b44602298f0960e900388cf3bb6043f645cf57ea9e3f5", + "sha256:54a07c09c586b0e4c619f02a5e94e36619da8e2b053e20f594348c0611803704" + ], + "index": "pypi", + "version": "==2017.7.27.1" + }, + "cffi": { + "hashes": [ + "sha256:040e0b5462c65a39b7112ad9e951f3f57b487d0db053ea510dcf8fc53931ab4b", + "sha256:2217d22cb001386947693bd93ca4370224989468daf2939f3bf717956fb0a119", + "sha256:42b38cd48e55b2a4c5f038f80137c2cb4393fd4e4d1c3dbeeb7aea9475b68ea1", + "sha256:45233398f5e7114e719d0b83ffd0642af7fbf05136195f500fb6b72133c8e998", + "sha256:47853d133f0de32697b55e0c2ee377d88e3143bc8645a3b93d38be2f5387b0a2", + "sha256:48d6589fda4ec0ccb7d973807890a733381f828d29167177eb7d97892255a769", + "sha256:4d2e388ce51e6a4efae8d4533ca15503ea957c41233bad165f8ccec354edca67", + "sha256:5152bbde6ee514a11e2cd1b76a7ccab5af1043356ec3df9efe0112d97b20dae3", + "sha256:56a8c39e6e7225474b36c16a03240fcbbfc5c8cb016645793e0dec2d2f9bdf67", + "sha256:59fc30a6f7331fe69cfc67ef151fd03a33491d25430ae0c4597e69c4235fa87e", + "sha256:5c20f34a6858cd8c59eac23f80bc80bb5fd294bd9b22b264255a3eec0ab459f5", + "sha256:5f4ff33371c6969b39b293d9771ee91e81d26f9129be093ca1b7be357fcefd15", + "sha256:6153b26756de2f34f80ca4e8b1e79011ad62bb670539dc3df71e62909a112fdf", + "sha256:6f1a2f85510c675b068e29625ff534f02567f24839cbd7d61cda6b7df62da2b1", + "sha256:765cf591a2f790b012075c7f54344a02b622a8561708ec27b3042c8c1699a465", + "sha256:7a55f044dd78fdca379d455550c581b9455af022b8f8b5d97f585a25f4da63f6", + "sha256:8a5632592b2752375b6d9e81da022896ece235713129fcc33703b3e10c639ef9", + "sha256:929d206ecfefadcec0b0e13956131eec7442829d240da020d67274fefb9e61a4", + "sha256:a76b89adecd46138aafdbd348563df1deb714962b5b3659ee1b6b0c5cf6ae0e5", + "sha256:b45efc8d81640a8e9122c1ba1817da2a998ca0168c40de7bbfc47b960771ca94", + "sha256:bd93a0df4dc72893ba604c05d487ea4d47a799e54172b89e414953685b0b08fb", + "sha256:c091a1ac521dac2770632f499c124762ddf978d23ced0472f19a8830e006eab8", + "sha256:d361e8c504bae02f665fad8d5ba4b74a6c1fab7862eec39cddcc21e4479fa275", + "sha256:d8930a719781987a876d6de8cf33ce937191c383158daafcb9db804bf3421adf", + "sha256:dd9577197fff563e4784ef4ace7da28c7da96a81895b69c008cdb82f1e82853c", + "sha256:ec1112466a924ba69df5a90f31d26d323e3b91ce21f94b10fa0b77f44a07df48", + "sha256:eddf6bd95b100f34c56a226847be92bde416d028213e87fd700bc593cbf3e9da", + "sha256:f14c53dc624529eef175c993ceb886f37ecece09825bf47bf3230aacbc3b7acc", + "sha256:f2ffdfd930bc4bcd4900def32fbb2a16a91656d9caafff573a25934327140431", + "sha256:f6202eab19fd2c44d7d7af2bb8d286cf18b7b3d67b0e5bf1f2f113c99303ff87", + "sha256:fd3c49d7d22ba75382e19c4877e1d4dca232ef48137442f8582c2259b0cdf612" + ], + "index": "pypi", + "version": "==1.11.0" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "index": "pypi", + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", + "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" + ], + "index": "pypi", + "version": "==6.7" + }, + "constantly": { + "hashes": [ + "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35", + "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d" + ], + "index": "pypi", + "version": "==15.1.0" + }, + "cryptography": { + "hashes": [ + "sha256:0f3dcdd06c5491cad2ed283ebc802805e3ac77ffe4789ad1b44ff0f0f891f59b", + "sha256:15d655434cf6f350462c6c97f24cbf42ee0e6270886a7937eaaa52fd09e474ce", + "sha256:1fb7dbf630daba53494da88de1e87429f2740e6ffdc92bbdcb385ddb9ff67b32", + "sha256:27b7b005debcf4c0562299d5ceeaea151120eaad7bd84ef91fb22c204a958f03", + "sha256:2f0b681e89530750e4c0adf98701cafd0384ac160ceafe39c9d10be6034d2cc7", + "sha256:317127295b14f5c943ed789ab60ade0d7d39041f8975d58a214af19dd1148705", + "sha256:369e4669cac8296e320f86d0a3c9f489b7593d2a144e7b0504b884bed56fd5d7", + "sha256:4ef205496c518aae7e699731dfff4b630c046ea9dbbe34e8ec8ce77bb399c1c7", + "sha256:4ffc706fbe4c12146050d432250896f2fcad8b77c6fd89d3c3bd456884dbf421", + "sha256:5544d9433f9c25d55d609e2b014d47c390288bb98ae84e35b8cf93e9f6a6c832", + "sha256:55cc98c93323efcf6f3300b6e9a7d435146a9d451ffd27aa14074f576f275786", + "sha256:59d2b285fb24d30deaf4143c2d048fcefdd7e560f30cd1745da881705ca965c6", + "sha256:5a266ff306683cd82b2f19203f469b7580aee115f1818d2b2d1e12f1ee7fd12b", + "sha256:5c9e46b219e2b65ab9192e10407d7698589794407f15f11f39abbf3b04d16890", + "sha256:795af466e52daf404d783877157b106bc79e0cffa55f8d6360fa9f9fd03148bd", + "sha256:8067fce92ec98cb52fabe2e10205ff27dba5755d9ffc1f521be9c504f9a6d5fc", + "sha256:886759453c5b93b1e8953e92abee65ee09fdf78bf655c95d0a206d6387adbfed", + "sha256:8bbff1de3002db5c7146b88fea392526260becc153adc8810103fc750c3eb333", + "sha256:a83318144a0037f264891ddfcbae9b75303949bb0efaece9713df5e04388b7ef", + "sha256:a972eb37b6409a2d67ddd956e6bf74f2e9315105083d1fdaa8bbe6eb520dc367", + "sha256:b17e4244212c9ec6526b6ad47534a12e0378387ad86228141053c112e759769d", + "sha256:b25d97585f1d2b2f16560737aba6257a10ca18aa3a2a70407c05fe8b64b375f9", + "sha256:bb715b0d8d5c9111e9656eabe58ee7b95842a797c04608aef7f52439ab73d5f3", + "sha256:bc86d3de2c7d63f3c73dff24fa43c55c00c7b8ea8102ed638c903c939b85e6a6", + "sha256:d04bb2425086c3fe86f7bc48915290b13e798497839fbb18ab7f6dffcf98cc3a", + "sha256:d32dad8448ea88fe8eb02dc943d46db658cf9e618386761e498802c79e97c937", + "sha256:d44db8e4435a10dcbf22fbc03072d18b31e6c1100084a2010239efd359f2f162", + "sha256:d5341dac22e8ab080e637c632d1b1e92acaa8e66a02d139c0c795c1797531384", + "sha256:fb7dfc4b3a79ca4b9233803b80617fd302374e57771c9a28e32ee6c1296cdea3", + "sha256:fe6d56a4ff08875d70cc9b336fcd6e938db5808cb47cbc33c6927affb659aded" + ], + "index": "pypi", + "version": "==2.0.3" + }, + "cssselect": { + "hashes": [ + "sha256:4f5f799a1d3182b04814007e9e7fc6c362f4489c7420d6b348cc901ece07ced9", + "sha256:73db1c054b9348409e2862fc6c0dde5c4e4fbe4da64c5c5a9e05fbea45744077" + ], + "index": "pypi", + "version": "==1.0.1" + }, + "enum34": { + "hashes": [ + "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", + "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", + "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", + "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" + ], + "index": "pypi", + "version": "==1.1.6" + }, + "future": { + "hashes": [ + "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + ], + "index": "pypi", + "version": "==0.16.0" + }, + "hyperlink": { + "hashes": [ + "sha256:1ec8e11fb4f5b330f25864bf8cfd3133dff1a3637dfd14fa441297df15fc7cf9", + "sha256:bc4ffdbde9bdad204d507bd8f554f16bba82dd356f6130cb16f41422909c33bc" + ], + "index": "pypi", + "version": "==17.3.1" + }, + "idna": { + "hashes": [ + "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f", + "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4" + ], + "index": "pypi", + "version": "==2.6" + }, + "incremental": { + "hashes": [ + "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f", + "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3" + ], + "index": "pypi", + "version": "==17.5.0" + }, + "ipaddress": { + "hashes": [ + "sha256:5d8534c8e185f2d8a1fda1ef73f2c8f4b23264e8e30063feeb9511d492a413e1", + "sha256:d34cf15d95ce9a734560f7400a8bd2ac2606f378e2a1d0eadbf1c98707e7c74a" + ], + "index": "pypi", + "version": "==1.0.18" + }, + "lxml": { + "hashes": [ + "sha256:0b8f3d6e669ea26849a6184f04c7802dbef6fd418a8b90e6c026e237db07af31", + "sha256:10c86b2248043f4428be33ed10202764b02b281eaa4550f16f0fbbc6ccaae9ac", + "sha256:1f81074e77c25f9b787fa3854f400ca924d3d798cb7ae910c0e7920be7138c90", + "sha256:3593f49858fc6229cd93326be06b099ae477fd65d8f4a981320a6d0bb7fc7a5a", + "sha256:48ab0e79175fd16f9478edc679ee14c79524c64b26f665f92cbecff81312d04d", + "sha256:52e18dd86f153c4383bb4c4ef62f81f9b7e44809d068848a5a183b2285496faf", + "sha256:550a51dee73c14e5863bdbbbe5836b2b8092a3f92631b5a908b9c41e72f123a5", + "sha256:567b76f291a8d02aa8b4d3f8295ae749ac4d532570d8a8c7176f0556c7d95891", + "sha256:61825daaf2d80dc3da7635ee108720b0739962db008343822753bbf343cbfd14", + "sha256:7030f44b758e930fd09ade87d770f5a231a19a8c561a3acc54e5122b5ec09e29", + "sha256:824664493a012f5b70396e841a4b4049bdaf445a70307e60f82fe35619f72cc7", + "sha256:88d137e440b5de35df2e0616da8e28a88d0119abdaa84520ad1ba815ee9da732", + "sha256:8996df6b0f205b00b89bbd04d88f1fa1e04139a025fd291aa4ddd05dc86836f4", + "sha256:9581b438e5d0d0a6fa3937fac2abffd95380bd513bcd39f6952bfcf20cf0b9a7", + "sha256:99b7eabfb46663ed5918eca4ed12420613ba24196964a741ccd962d09296c0b2", + "sha256:9f399c37b8e61c3989ef12ecf0abd9c10a5075f0fc9ad1ecd67ce6f9c72a7211", + "sha256:a211288459c9115ddb255ff88e8ac12dc2063e70bddc15e3c65136477a358bb5", + "sha256:a8ad0adeedbbb7b85916214fcd4f5d02829d0e7b3c32abc298789218b6c3d699", + "sha256:b2ecb3fd5470b740dfc21b064bbc1337be4b7b805994a868488145d36f35f517", + "sha256:b7f6ef610680709be11cb7662e46e254bc561dafe0de3b4848be2cf3924bd300", + "sha256:c446fde3284c363cd2085ad1ce5a07c18f15f6766d72684622bc14b0a9ddfd29", + "sha256:c4e02657e629f02ab8712471d77d6896c2cf6f09f8ffa6a0f23b1b1ef0318474", + "sha256:d4507916c408feec2ea8cee3f0d1380e49ea431f6e07b0dd927388bd6e92d6eb", + "sha256:d78c0a114cf127a41a526aef99aef539c0b2537e57f04a2cc7a49e2c94a44ab8", + "sha256:d9ec728caddb161405e7c33ed9d176e96309893481370163bbf4b00e43008795", + "sha256:e37eda3d05519918403084b43eb7324df21a0daf45c8ae8172a860710dd0fa78", + "sha256:e908d685800626f10cd6ae01a013fc42094be167fb2a683eb920dfddfaa0ee76", + "sha256:f7bc9f702500e205b1560d620f14015fec76dcd6f9e889a946a2ddcc3c344fd0" + ], + "index": "pypi", + "version": "==4.0.0" + }, + "ndg-httpsclient": { + "hashes": [ + "sha256:28d62f2fc93dcd20793ddf2ce9ec1ab9f3458a11a784105a6d67f218ca53de63", + "sha256:5b1e1b1991990487e28e5b95e062c9d06f8f00e3cd79eaddc448ce06c5e3b776", + "sha256:7bfd8c5cfcbc241a93ca6a4e45f952650f5c7ecf7c49b1dbcf5f4d390240be0b" + ], + "index": "pypi", + "version": "==0.4.3" + }, + "ordereddict": { + "hashes": [ + "sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f" + ], + "index": "pypi", + "version": "==1.1" + }, + "parsel": { + "hashes": [ + "sha256:1c5a31cb3f0fdca3d19f2ef966d595df69181c2d81b6c56582732c819e2f4e26", + "sha256:29eb4bd74e22e41138ea8ed96ce8e477b8116f97a13a991e39cb150fdde7eabd" + ], + "index": "pypi", + "version": "==1.2.0" + }, + "pyasn1": { + "hashes": [ + "sha256:90a6dfe991beb1f6240bc19e9fda02f513517523e3b96474a0887a94022dbad0", + "sha256:c7a1186416bcdaf19b56f5b2506200306d7d4cf222d3d7c2d4cb600ccacdaf1b" + ], + "index": "pypi", + "version": "==0.3.5" + }, + "pyasn1-modules": { + "hashes": [ + "sha256:529d5d509562c22877a53771c5b394d8102c98cd62f33bb5af23b475879ea6d5", + "sha256:b07c17bdb34d6f64aafea6269f2e8fb306a57473f0f38d9a6ca389d6ab30ac4a" + ], + "index": "pypi", + "version": "==0.1.4" + }, + "pycparser": { + "hashes": [ + "sha256:99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226" + ], + "index": "pypi", + "version": "==2.18" + }, + "pydispatcher": { + "hashes": [ + "sha256:5570069e1b1769af1fe481de6dd1d3a388492acddd2cdad7a3bde145615d5caf", + "sha256:5be4a8be12805ef7d712dd9a93284fb8bc53f309867e573f653a72e5fd10e433" + ], + "index": "pypi", + "version": "==2.0.5" + }, + "pyopenssl": { + "hashes": [ + "sha256:29630b9064a82e04d8242ea01d7c93d70ec320f5e3ed48e95fcabc6b1d0f6c76", + "sha256:aade9985b93eaec51b0c0a2a60d14bb8dcff1ff8e36fe542e3c22812ec07315e" + ], + "index": "pypi", + "version": "==17.3.0" + }, + "pyperclip": { + "hashes": [ + "sha256:a3cb6df5d8f1557ca8fc514d94fabf50dc5a97042c90e5ba4f3611864fed3fc5" + ], + "index": "pypi", + "version": "==1.5.27" + }, + "python-dotenv": { + "hashes": [ + "sha256:45e927c34204c90f5faa35ea8709b894f6b1a7712d77eb50940601068040993b", + "sha256:dc7940052cfe170e881aea40feb4ea7776e6a97170ed038fd2ee7e26e47585f2" + ], + "index": "pypi", + "version": "==0.7.1" + }, + "queuelib": { + "hashes": [ + "sha256:a6829918157ed433fafa87b0bb1e93e3e63c885270166db5884a02c34c86f914", + "sha256:c97f2120e070e2b8a5e8a60afe10bd7b3ae70c0994d6bb9c4d802b0448241339" + ], + "index": "pypi", + "version": "==1.4.2" + }, + "ratelimit": { + "hashes": [ + "sha256:d001f4dacf4880f64dafb5bc6ef0cc882aa94f291c623bbcf080e0a882e62351" + ], + "index": "pypi", + "version": "==1.4.1" + }, + "requests": { + "hashes": [ + "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", + "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" + ], + "index": "pypi", + "version": "==2.18.4" + }, + "requests-file": { + "hashes": [ + "sha256:f518e7cfe048e053fd1019dfb891b4c55b871c56c5a31693d733240c80b8f191" + ], + "index": "pypi", + "version": "==1.4.2" + }, + "scrapy": { + "hashes": [ + "sha256:08d86737c560dcc1c4b73ac0ac5bd8d14b3e2265c1f7b195f0b73ab13741fe03", + "sha256:31a0bf05d43198afaf3acfb9b4fb0c09c1d7d7ff641e58c66e36117f26c4b755" + ], + "index": "pypi", + "version": "==1.5.0" + }, + "selenium": { + "hashes": [ + "sha256:5071f43daa2e698d60d5633ab0a6630cc68a852b360be99144f1c4c1ace2746c", + "sha256:f507181f13768d73b98dd9647a466ea5758ef5c7f07b62a285d2bd8de9b27016" + ], + "index": "pypi", + "version": "==2.53.6" + }, + "service-identity": { + "hashes": [ + "sha256:0e76f3c042cc0f5c7e6da002cf646f59dc4023962d1d1166343ce53bdad39e17", + "sha256:4001fbb3da19e0df22c47a06d29681a398473af4aa9d745eca525b3b2c2302ab" + ], + "index": "pypi", + "version": "==17.0.0" + }, + "six": { + "hashes": [ + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + ], + "index": "pypi", + "version": "==1.11.0" + }, + "slacker": { + "hashes": [ + "sha256:5d54950a1956c6faff3c82cd116aed4436cf0e322a72747d77b9189d5ec7f8cb" + ], + "index": "pypi", + "version": "==0.9.60" + }, + "tldextract": { + "hashes": [ + "sha256:6a705710f3a1d5ce8a084c888db7bad7aff1dcdf27962f7c7f7058cc42349212", + "sha256:796d86aa22c382770b28db45547b298cbf855dcdcd8e1c7264419d96c0f2381a" + ], + "index": "pypi", + "version": "==2.1.0" + }, + "twisted": { + "hashes": [ + "sha256:60cfb24a20a14d8113bbbdc15601da895f6c025189cb660b6dd1e73623675419", + "sha256:f198a494f0df2482f7c5f99d7f3eef33d22763ffc76641b36fec476b878002ea" + ], + "index": "pypi", + "version": "==17.5.0" + }, + "urllib3": { + "hashes": [ + "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", + "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" + ], + "index": "pypi", + "version": "==1.22" + }, + "w3lib": { + "hashes": [ + "sha256:7b661935805b7d39afe90bb32dec8e4d20b377e74c66597eb1ddfad10071938e", + "sha256:c48731d5d73cde86f9c3c2bd6898d165f670120427353a7c8f9d6c685561d3c4" + ], + "index": "pypi", + "version": "==1.18.0" + }, + "zope-interface": { + "hashes": [ + "sha256:0b84b11b828770f9d711be99d037bb076ed0291a343fd910b683a9bd7866e0f2", + "sha256:0d1e1f3a816b5b4774c0a7cf46adf399d298f44c4de71ecab6ea83b9b71b9bc7", + "sha256:0e9fa25e4786d02e88a17f0b1152ab3e901ab47a44b05a6c711e253836d4698c", + "sha256:0effab9d51933e35ccd01f4c8c6d72f8040df78deacb2ec9176a7519722ce9ba", + "sha256:13119a6ab5cd1d9b28b85ac9c7e1dadd4e0b743016a5b77a374ce51b3d26f339", + "sha256:17873e481cd2dce1db4fc15bbcb59400c100771354bddad8302fadb7dd2a0f0e", + "sha256:21c506b586ab17e7d353e5597683112285b0889330c9bd4f374e4f8b387d3348", + "sha256:2bcd66cc16dccf8b1cca37abe8ce947d38b54f78c244fd45102fc00997c7446a", + "sha256:3c87138213b2a3896adcbf4fcc3dd916c68cef405cadcb9327d0f08949e10194", + "sha256:40026b28cbf065ae54dcfdb1f95f14d6d07c4f38a4293a3bb663f2bee620fd4e", + "sha256:5c14d72a86df23bc473efa8567a2f04fc94d1ba84a8ba651e3a9991e8b20a15a", + "sha256:5e0c85069a508835b192d5fd304aef4a7b23a325f2cd58e8953ae1c1eb1a8570", + "sha256:6c9e61ac24ed84d9c6a20155d979bd08f278c76450d7ab65a107057780457aac", + "sha256:74eba4f1d2100424d51956103836d7c07bc8e5bf17fcf34a59b784e1fcedeaf2", + "sha256:74f2a8c466124f852cb78bbb6bf8306340af1885e77870abc5418cbfeb54fece", + "sha256:7cc6f03a905094983f92cd6a97244de05e9293f7b09e110ce4c7c81de8a26a3a", + "sha256:8aa8f86fb2c7a7cab830fc871a2bd19af18dabb0e940ec9adb658136eb12ebae", + "sha256:8adee390aaeeae7d5858f023e552bdb631ff4f129900ea1e8bb62c3722af12f5", + "sha256:9a567f17f3f41e955bc87cfa39cbd5713ba3077dfd8a06a360605ec45407e357", + "sha256:aeeff785ba7cd5cad4b2cfd2930bf5c2431e0fdfdf7ffc0b3af44827f1007611", + "sha256:b2467ebf96561f022265cf0e1920d19ab1178deb3f0bb9fbd81bd90e995251c1", + "sha256:c55003bf2ab1046dbc0e79a4a9ac7af2f5800ed6ec4cb8e35fc3c99a21f380a8", + "sha256:dac9c7d45f8ab94f6f1a0dd167398a633924c85238cc8d76a6a04e6eac04def7", + "sha256:f7cbfd9698c16cb2ed164e7b9e4b3a24217e51fccfd95c3fd3dcca9bc1e7c9e9", + "sha256:fae7b143aea7248cbbed9ed3977e8198fa82805d1643556b03d23df28ffdf99b", + "sha256:fbe4140cc7743c541c302a0bf31dca432b0c0a914053809a9338237d54c3b6ca", + "sha256:fd56bbded257c0da54e98dce7a030e76cc018b8c3eaeb8ce8f35487fb1c859e3" + ], + "index": "pypi", + "version": "==4.4.2" + }, + "zope.interface": { + "hashes": [ + "sha256:086707e0f413ff8800d9c4bc26e174f7ee4c9c8b0302fbad68d083071822316c", + "sha256:1157b1ec2a1f5bf45668421e3955c60c610e31913cc695b407a574efdbae1f7b", + "sha256:11ebddf765bff3bbe8dbce10c86884d87f90ed66ee410a7e6c392086e2c63d02", + "sha256:14b242d53f6f35c2d07aa2c0e13ccb710392bcd203e1b82a1828d216f6f6b11f", + "sha256:1b3d0dcabc7c90b470e59e38a9acaa361be43b3a6ea644c0063951964717f0e5", + "sha256:20a12ab46a7e72b89ce0671e7d7a6c3c1ca2c2766ac98112f78c5bddaa6e4375", + "sha256:298f82c0ab1b182bd1f34f347ea97dde0fffb9ecf850ecf7f8904b8442a07487", + "sha256:2f6175722da6f23dbfc76c26c241b67b020e1e83ec7fe93c9e5d3dd18667ada2", + "sha256:3b877de633a0f6d81b600624ff9137312d8b1d0f517064dfc39999352ab659f0", + "sha256:4265681e77f5ac5bac0905812b828c9fe1ce80c6f3e3f8574acfb5643aeabc5b", + "sha256:550695c4e7313555549aa1cdb978dc9413d61307531f123558e438871a883d63", + "sha256:5f4d42baed3a14c290a078e2696c5f565501abde1b2f3f1a1c0a94fbf6fbcc39", + "sha256:62dd71dbed8cc6a18379700701d959307823b3b2451bdc018594c48956ace745", + "sha256:7040547e5b882349c0a2cc9b50674b1745db551f330746af434aad4f09fba2cc", + "sha256:7e099fde2cce8b29434684f82977db4e24f0efa8b0508179fce1602d103296a2", + "sha256:7e5c9a5012b2b33e87980cee7d1c82412b2ebabcb5862d53413ba1a2cfde23aa", + "sha256:81295629128f929e73be4ccfdd943a0906e5fe3cdb0d43ff1e5144d16fbb52b1", + "sha256:95cc574b0b83b85be9917d37cd2fad0ce5a0d21b024e1a5804d044aabea636fc", + "sha256:968d5c5702da15c5bf8e4a6e4b67a4d92164e334e9c0b6acf080106678230b98", + "sha256:9e998ba87df77a85c7bed53240a7257afe51a07ee6bc3445a0bf841886da0b97", + "sha256:a0c39e2535a7e9c195af956610dba5a1073071d2d85e9d2e5d789463f63e52ab", + "sha256:a15e75d284178afe529a536b0e8b28b7e107ef39626a7809b4ee64ff3abc9127", + "sha256:a6a6ff82f5f9b9702478035d8f6fb6903885653bff7ec3a1e011edc9b1a7168d", + "sha256:b639f72b95389620c1f881d94739c614d385406ab1d6926a9ffe1c8abbea23fe", + "sha256:bad44274b151d46619a7567010f7cde23a908c6faa84b97598fd2f474a0c6891", + "sha256:bbcef00d09a30948756c5968863316c949d9cedbc7aabac5e8f0ffbdb632e5f1", + "sha256:d788a3999014ddf416f2dc454efa4a5dbeda657c6aba031cf363741273804c6b", + "sha256:eed88ae03e1ef3a75a0e96a55a99d7937ed03e53d0cffc2451c208db445a2966", + "sha256:f99451f3a579e73b5dd58b1b08d1179791d49084371d9a47baad3b22417f0317" + ], + "version": "==4.6.0" + } + }, + "develop": {} +} From 493712aadf8a753aac03c1fe14e1d486bdc75162 Mon Sep 17 00:00:00 2001 From: s-pace Date: Fri, 2 Nov 2018 18:20:31 +0100 Subject: [PATCH 16/17] give instructions regarding how to use our scraper without docker closes algolia/docsearch#449 --- CONTRIBUTING.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f5de178..79c33dfd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,16 @@ Hello you, + +## Install the scrapper without Docker + +We do recommend the use of [pipenv][1] to install the whole python environment + +- [Install pipenv][2] +- `pipenv --python 2.7 install` + +You should be ready to go. + + ## Run the tests To run the full test suite, run `./docsearch test`. Note that it requires that @@ -42,4 +53,6 @@ git clone git@github.com:algolia/docsearch-configs.git configs/public ``` The CLI will then have more commands for you to run. +[1]: https://github.com/pypa/pipenv +[2]: https://pipenv.readthedocs.io/en/latest/install/#installing-pipenv From c0dae0c87348e88d5d8772d829f42f68ef0754ca Mon Sep 17 00:00:00 2001 From: s-pace Date: Fri, 2 Nov 2018 18:23:11 +0100 Subject: [PATCH 17/17] format .md --- CONTRIBUTING.md | 8 ++++---- README.md | 1 - cli/oh-my-zsh/README.md | 4 ++-- deployer/README.md | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 79c33dfd..cbffbbb0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,5 @@ Hello you, - ## Install the scrapper without Docker We do recommend the use of [pipenv][1] to install the whole python environment @@ -10,7 +9,6 @@ We do recommend the use of [pipenv][1] to install the whole python environment You should be ready to go. - ## Run the tests To run the full test suite, run `./docsearch test`. Note that it requires that @@ -47,12 +45,14 @@ DEPLOY_KEY= Ping the @docsearch team to get access to those credentials. ## Ready to deploy + Clone the configurations repo in your docsearch-scraper directory. Command to run from the scraper root: + ```bash git clone git@github.com:algolia/docsearch-configs.git configs/public ``` -The CLI will then have more commands for you to run. + +The CLI will then have more commands for you to run. [1]: https://github.com/pypa/pipenv [2]: https://pipenv.readthedocs.io/en/latest/install/#installing-pipenv - diff --git a/README.md b/README.md index e64b61c0..153b315f 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,6 @@ DocSearch is made of 3 repositories: from your documentation. The code is open-source and you can run it from a Docker image - [1]: https://community.algolia.com/docsearch/ [2]: https://community.algolia.com/docsearch/run-your-own.html [3]: https://github.com/algolia/docsearch diff --git a/cli/oh-my-zsh/README.md b/cli/oh-my-zsh/README.md index 84d1662c..6941b7ca 100644 --- a/cli/oh-my-zsh/README.md +++ b/cli/oh-my-zsh/README.md @@ -1,4 +1,4 @@ ## How to install -- copy the ```docsearch``` folder in ```~/.oh-my-zsh/custom/plugins/``` -- enable the plugin in your ```zshrc``` by modify the plugins variable: ``plugins=([...], docsearch) \ No newline at end of file +- copy the `docsearch` folder in `~/.oh-my-zsh/custom/plugins/` +- enable the plugin in your `zshrc` by modify the plugins variable: ``plugins=([...], docsearch) diff --git a/deployer/README.md b/deployer/README.md index c3742f08..7989ab63 100644 --- a/deployer/README.md +++ b/deployer/README.md @@ -6,4 +6,4 @@ API_KEY="" \ WEBSITE_USERNAME="" \ WEBSITE_PASSWORD="" \ ./deploy -``` \ No newline at end of file +```