Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 22 additions & 11 deletions gidgethub/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
__version__ = "5.4.0.dev"

import http
from typing import Any, Optional
from typing import Any, Optional, Mapping


class GitHubException(Exception):
Expand All @@ -19,8 +19,9 @@ class ValidationFailure(GitHubException):
class HTTPException(GitHubException):
"""A general exception to represent HTTP responses."""

def __init__(self, status_code: http.HTTPStatus, *args: Any) -> None:
def __init__(self, status_code: http.HTTPStatus, *args: Any, headers: Mapping[str, str] =None) -> None:
self.status_code = status_code
self.headers = headers or {}
if args:
super().__init__(*args)
else:
Expand All @@ -43,23 +44,33 @@ class BadRequest(HTTPException):
class BadRequestUnknownError(BadRequest):
"""A bad request whose response body is not JSON."""

def __init__(self, response: str) -> None:
def __init__(self, response: str, **kwargs) -> None:
self.response = response
super().__init__(http.HTTPStatus.UNPROCESSABLE_ENTITY)
super().__init__(http.HTTPStatus.UNPROCESSABLE_ENTITY, **kwargs)


class RateLimitExceeded(BadRequest):
"""Request rejected due to the rate limit being exceeded."""

# Technically rate_limit is of type gidgethub.sansio.RateLimit, but a
# circular import comes about if you try to properly declare it.
def __init__(self, rate_limit: Any, *args: Any) -> None:
def __init__(self, rate_limit: Any, *args: Any, **kwargs) -> None:
self.rate_limit = rate_limit

if not args:
super().__init__(http.HTTPStatus.FORBIDDEN, "rate limit exceeded")
super().__init__(http.HTTPStatus.FORBIDDEN, "rate limit exceeded", **kwargs)
else:
super().__init__(http.HTTPStatus.FORBIDDEN, *args)
super().__init__(http.HTTPStatus.FORBIDDEN, *args, **kwargs)


class SecondaryRateLimitExceeded(BadRequest):
"""Request rejected due to the secondary rate limit being exceeded."""
def __init__(self, status_code: http.HTTPStatus, *args: Any, headers: Mapping[str, str] =None) -> None:

if not args:
super().__init__(status_code, "secondary rate limit exceeded", headers=headers)
else:
super().__init__(status_code, *args, headers=headers)


class InvalidField(BadRequest):
Expand All @@ -69,10 +80,10 @@ class InvalidField(BadRequest):
invalid are stored in the errors attribute.
"""

def __init__(self, errors: Any, *args: Any) -> None:
def __init__(self, errors: Any, *args: Any, **kwargs) -> None:
"""Store the error details."""
self.errors = errors
super().__init__(http.HTTPStatus.UNPROCESSABLE_ENTITY, *args)
super().__init__(http.HTTPStatus.UNPROCESSABLE_ENTITY, *args, **kwargs)


class ValidationError(BadRequest):
Expand All @@ -82,10 +93,10 @@ class ValidationError(BadRequest):
are stored in the *errors* attribute.
"""

def __init__(self, errors: Any, *args: Any) -> None:
def __init__(self, errors: Any, *args: Any, **kwargs) -> None:
"""Store the error details."""
self.errors = errors
super().__init__(http.HTTPStatus.UNPROCESSABLE_ENTITY, *args)
super().__init__(http.HTTPStatus.UNPROCESSABLE_ENTITY, *args, **kwargs)


class GitHubBroken(HTTPException):
Expand Down
13 changes: 12 additions & 1 deletion gidgethub/abc.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import abc
import http
import json
import os
from typing import Any, AsyncGenerator, Dict, Mapping, MutableMapping, Optional, Tuple
from typing import Optional as Opt

Expand All @@ -18,7 +19,7 @@
GraphQLResponseTypeError,
)
from . import sansio

from .sansio import secondary_rate_limit_retry

# Value represents etag, last-modified, data, and next page.
CACHE_TYPE = MutableMapping[str, Tuple[Opt[str], Opt[str], Any, Opt[str]]]
Expand All @@ -29,6 +30,11 @@
ITERABLE_KEY = "items"


MANAGE_SECONDARY_RATE_LIMIT = "GIDGETHUB_MANAGE_SECONDARY_RATE_LIMIT" in os.environ
SECONDARY_RATE_LIMIT_RETRIES = os.environ.get("GIDGETHUB_SECONDARY_RATE_LIMIT_RETRY", 3)
SECONDARY_RATE_LIMIT_BASE_DELAYS = os.environ.get("GIDGETHUB_SECONDARY_RATE_LIMIT_BASE_DELAY", 60)


class GitHubAPI(abc.ABC):
"""Provide an idiomatic API for making calls to GitHub's API."""

Expand Down Expand Up @@ -56,6 +62,11 @@ async def _request(
async def sleep(self, seconds: float) -> None:
"""Sleep for the specified number of seconds."""

@secondary_rate_limit_retry(
max_retries=SECONDARY_RATE_LIMIT_RETRIES,
base_delay=SECONDARY_RATE_LIMIT_BASE_DELAYS,
wrapped=MANAGE_SECONDARY_RATE_LIMIT,
)
async def _make_request(
self,
method: str,
Expand Down
81 changes: 75 additions & 6 deletions gidgethub/sansio.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,18 @@
"""

import datetime
import time
from email.message import Message
import hmac
import http
import json
import re
from typing import Any, Dict, Mapping, Optional, Tuple, Type, Union
from functools import wraps
from typing import Any, Dict, Mapping, Optional, Tuple, Type, Union, Callable, ParamSpec, TypeVar
import urllib.parse
import logging

logger = logging.getLogger(__name__)

import uritemplate
from uritemplate import variable
Expand All @@ -27,7 +32,7 @@
RateLimitExceeded,
RedirectionException,
ValidationError,
ValidationFailure,
ValidationFailure, SecondaryRateLimitExceeded,
)


Expand Down Expand Up @@ -141,6 +146,7 @@ def from_http(
"expected a content-type of "
"'application/json' or "
"'application/x-www-form-urlencoded'",
headers=headers
) from exc
return cls(
data,
Expand Down Expand Up @@ -282,6 +288,69 @@ def from_http(cls, headers: Mapping[str, str]) -> Optional["RateLimit"]:
return cls(limit=limit, remaining=remaining, reset_epoch=reset_epoch)


P = ParamSpec('P')
T = TypeVar('T')


def secondary_rate_limit_retry(max_retries: int, base_delay: int, wrapped: bool = True) -> Callable[[Callable[P, T]], Callable[P, T]]:
def decorator(func: Callable[P, T]) -> Callable[P, T]:
@wraps(func)
def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
retries = 0

while retries <= max_retries:
try:
return func(*args, **kwargs)
except BadRequest as e:
if e.status_code not in (403, 429):
raise

if retries >= max_retries:
raise SecondaryRateLimitExceeded(
http.HTTPStatus(e.status_code),
headers=e.headers,
) from e

retry_after = e.headers.get("retry-after") or e.headers.get("Retry-After")
reset_time = e.headers.get("x-ratelimit-reset") or e.headers.get("X-RateLimit-Reset")

if retry_after:
delay = int(retry_after)
logger.warning(
f"Secondary rate limit hit. Retrying after {delay} seconds "
f"(attempt {retries + 1}/{max_retries})"
)
elif reset_time:
current_time = int(time.time())
delay = max(int(reset_time) - current_time, 0)
logger.warning(
f"Primary rate limit hit. Retrying after {delay} seconds "
f"(attempt {retries + 1}/{max_retries})"
)
else:
delay = base_delay * (2 ** retries)
logger.warning(
f"Rate limit hit without retry information. "
f"Using exponential backoff: {delay} seconds "
f"(attempt {retries + 1}/{max_retries})"
)

logger.info(
f"Rate limit error details - Status: {e.status_code}, "
f"Headers: {dict(e.headers)}"
)

time.sleep(delay)
retries += 1
return None

if wrapped:
return wrapper
else:
return func
return decorator


_link_re = re.compile(
r"\<(?P<uri>[^>]+)\>;\s*" r'(?P<param_type>\w+)="(?P<param_value>\w+)"(,\s*)?'
)
Expand Down Expand Up @@ -339,13 +408,13 @@ def decipher_response(
if status_code == 403:
rate_limit = RateLimit.from_http(headers)
if rate_limit and not rate_limit.remaining:
raise RateLimitExceeded(rate_limit, message)
raise RateLimitExceeded(rate_limit, message, headers=headers)
elif status_code == 422:
try:
errors = data.get("errors", None)
except AttributeError:
# Not JSON so don't know why the request failed.
raise BadRequestUnknownError(data)
raise BadRequestUnknownError(data, headers=headers)
exc_type = InvalidField
if errors:
if isinstance(errors, str):
Expand All @@ -366,7 +435,7 @@ def decipher_response(
message = f"{message}: {error_context}"
else:
message = data["message"]
raise exc_type(errors, message)
raise exc_type(errors, message, headers=headers)
elif status_code >= 300:
exc_type = RedirectionException
else:
Expand All @@ -377,7 +446,7 @@ def decipher_response(
args = status_code_enum, message
else:
args = (status_code_enum,)
raise exc_type(*args)
raise exc_type(*args, headers=headers)


DOMAIN = "https://api.github.com"
Expand Down
11 changes: 11 additions & 0 deletions tests/test_sansio.py
Original file line number Diff line number Diff line change
Expand Up @@ -286,12 +286,14 @@ def test_5XX(self):
with pytest.raises(GitHubBroken) as exc_info:
sansio.decipher_response(status_code, {}, b"")
assert exc_info.value.status_code == http.HTTPStatus(status_code)
assert exc_info.value.headers == {}

def test_4XX_no_message(self):
status_code = 400
with pytest.raises(BadRequest) as exc_info:
sansio.decipher_response(status_code, {}, b"")
assert exc_info.value.status_code == http.HTTPStatus(status_code)
assert exc_info.value.headers == {}

def test_4XX_message(self):
status_code = 400
Expand All @@ -301,6 +303,7 @@ def test_4XX_message(self):
sansio.decipher_response(status_code, headers, message)
assert exc_info.value.status_code == http.HTTPStatus(status_code)
assert str(exc_info.value) == "it went bad"
assert exc_info.value.headers == headers

def test_404(self):
status_code = 404
Expand All @@ -309,6 +312,7 @@ def test_404(self):
sansio.decipher_response(status_code, headers, body)
assert exc_info.value.status_code == http.HTTPStatus(status_code)
assert str(exc_info.value) == "Not Found"
assert exc_info.value.headers == headers

def test_403_rate_limit_exceeded(self):
status_code = 403
Expand All @@ -322,6 +326,7 @@ def test_403_rate_limit_exceeded(self):
with pytest.raises(RateLimitExceeded) as exc_info:
sansio.decipher_response(status_code, headers, body)
assert exc_info.value.status_code == http.HTTPStatus(status_code)
assert exc_info.value.headers == headers

def test_403_forbidden(self):
status_code = 403
Expand All @@ -334,6 +339,7 @@ def test_403_forbidden(self):
with pytest.raises(BadRequest) as exc_info:
sansio.decipher_response(status_code, headers, b"")
assert exc_info.value.status_code == http.HTTPStatus(status_code)
assert exc_info.value.headers == headers

def test_422(self):
status_code = 422
Expand All @@ -345,6 +351,7 @@ def test_422(self):
sansio.decipher_response(status_code, headers, body)
assert exc_info.value.status_code == http.HTTPStatus(status_code)
assert str(exc_info.value) == "it went bad for 'title'"
assert exc_info.value.headers == headers

def test_422_custom_code(self):
status_code = 422
Expand All @@ -365,6 +372,7 @@ def test_422_custom_code(self):
str(exc_info.value)
== "it went bad: 'A pull request already exists for foo:1.'"
)
assert exc_info.value.headers == headers

def test_422_errors_as_string(self):
"""Test 422 response where 'errors' field is a string instead of list of objects."""
Expand All @@ -386,6 +394,7 @@ def test_422_errors_as_string(self):
str(exc_info.value)
== "Validation Failed: Validation failed: This SHA and context has reached the maximum number of statuses."
)
assert exc_info.value.headers == headers

def test_422_no_errors_object(self):
status_code = 422
Expand All @@ -401,6 +410,7 @@ def test_422_no_errors_object(self):
sansio.decipher_response(status_code, headers, body)
assert exc_info.value.status_code == http.HTTPStatus(status_code)
assert str(exc_info.value) == "Reference does not exist"
assert exc_info.value.headers == headers

def test_422_html_response(self):
# https://github.com/brettcannon/gidgethub/issues/81
Expand All @@ -412,6 +422,7 @@ def test_422_html_response(self):
sansio.decipher_response(status_code, headers, encoded_body)
assert exc_info.value.status_code == http.HTTPStatus(status_code)
assert exc_info.value.response == body
assert exc_info.value.headers == headers

def test_3XX(self):
status_code = 301
Expand Down