Merge pull request #110 from yan12125/investigate-crates-io-403

Fix 403 errors from crates.io

.travis.yml

@@ -7,8 +7,8 @@ python:
   - "3.6"
   - "3.7"
   - "nightly"
-  - "pypy3.5"
-install: pip install -U $DEPS pytest pytest-asyncio flaky structlog
+  - "pypy3.6-7.1.1"
+install: pip install -U $DEPS pytest pytest-asyncio pytest-httpbin flaky structlog
 script: pytest
 env:
   global:
@@ -26,5 +26,10 @@ matrix:
   fast_finish: true
   allow_failures:
     # doesn't work well, see https://travis-ci.org/lilydjwg/nvchecker/jobs/376326582
-    - python: pypy3.5
+    - python: pypy3.6-7.1.1
       env: DEPS=aiohttp
+
+addons:
+  apt:
+    packages:
+    - libgnutls-dev

nvchecker/source/aiohttp_httpclient.py

@@ -4,6 +4,8 @@
 import atexit
 import asyncio
 import aiohttp
+from .httpclient import DEFAULT_USER_AGENT
+
 connector = aiohttp.TCPConnector(limit=20)
 
 __all__ = ['session', 'HTTPError', 'NetworkErrors']
@@ -19,6 +21,8 @@ async def _request(self, *args, **kwargs):
         if hasattr(self, "nv_config") and self.nv_config.get("proxy"):
             kwargs.setdefault("proxy", self.nv_config.get("proxy"))
 
+        kwargs.setdefault("headers", {}).setdefault('User-Agent', DEFAULT_USER_AGENT)
+
         res = await super(BetterClientSession, self)._request(
             *args, **kwargs)
         if res.status >= 400:

nvchecker/source/github.py

@@ -46,7 +46,6 @@ async def get_version_real(name, conf, **kwargs):
     url += '?' + urlencode(parameters)
   headers = {
     'Accept': 'application/vnd.github.quicksilver-preview+json',
-    'User-Agent': 'lilydjwg/nvchecker',
   }
   if 'NVCHECKER_GITHUB_TOKEN' in os.environ:
     headers['Authorization'] = 'token %s' % os.environ['NVCHECKER_GITHUB_TOKEN']

nvchecker/source/httpclient.py

@@ -0,0 +1 @@
+DEFAULT_USER_AGENT = 'lilydjwg/nvchecker'

nvchecker/source/tornado_httpclient.py

@@ -15,6 +15,8 @@
 except ImportError:
   pycurl = None
 
+from .httpclient import DEFAULT_USER_AGENT
+
 __all__ = ['session', 'HTTPError', 'NetworkErrors']
 
 client = AsyncHTTPClient()
@@ -51,6 +53,7 @@ def get(self, url, **kwargs):
       q = urlencode(params)
       url += '?' + q
 
+    kwargs.setdefault("headers", {}).setdefault('User-Agent', DEFAULT_USER_AGENT)
     r = HTTPRequest(url, **kwargs)
     return ResponseManager(r)
 

setup.py

@@ -28,6 +28,7 @@
   tests_require = [
     'pytest',
     'pytest-asyncio',
+    'pytest-httpbin',
     'flaky',
   ],
   entry_points = {

tests/test_regex.py

@@ -4,12 +4,16 @@
 import pytest
 pytestmark = pytest.mark.asyncio
 
-@pytest.mark.skipif(True,
-                    reason='httpbin is overloaded?')
-async def test_regex_httpbin(get_version):
+async def test_regex_httpbin_default_user_agent(get_version, httpbin):
     assert await get_version("example", {
-        "url": "https://httpbin.org/get",
-        "regex": r'"User-Agent": "(\w+)"',
+        "url": httpbin.url + "/get",
+        "regex": r'"User-Agent":\s*"([^"]+)"',
+    }) == "lilydjwg/nvchecker"
+
+async def test_regex_httpbin(get_version, httpbin):
+    assert await get_version("example", {
+        "url": httpbin.url + "/get",
+        "regex": r'"User-Agent":\s*"([^"]+)"',
         "user_agent": "Meow",
     }) == "Meow"