import asyncio
import aiohttp
import logging
import requests
from pyeudiw.tools.exceptions import HttpError
from pyeudiw.jwt.utils import decode_jwt_payload, is_jwt_format
DEFAULT_HTTPC_PARAMS = {
"connection": {"ssl": False}, # FALSE FOR DEV ENVIRONMENT ONLY
"session": {"timeout": 4},
}
[docs]
def http_get(
url: str, httpc_params: dict = DEFAULT_HTTPC_PARAMS, decode_output: bool = True
) -> str:
"""
Perform a GET http call.
:param urls: the url list where fetch the content
:type urls: list[str]
:param httpc_params: parameters to perform http requests.
:type httpc_params: dict
:raises HttpError: if the response status code is not 200 or a connection error occurs
:returns: the list of responses
:rtype: list[requests.Response] | list[str]
"""
_conf = {
"verify": httpc_params["connection"]["ssl"],
"timeout": httpc_params["session"]["timeout"],
}
if not _conf["verify"]:
logging.warning(f"SSL certificate verification is disabled for HTTP request to: {url}")
try:
# nosec B113: timeout is set via _conf["timeout"], which Bandit cannot detect through **_conf unpacking
response = requests.get(url, **_conf) # nosec B113
except requests.exceptions.ConnectionError as e:
raise HttpError(f"Connection error: {e}")
if response.status_code != 200:
raise HttpError(f"HTTP error: {response.status_code} -- {response.reason}")
output = response.content
if isinstance(output, bytes):
output = output.decode("utf-8")
if decode_output:
if is_jwt_format(output):
output = decode_jwt_payload(output)
else:
output = response.json()
return output
[docs]
def http_get_sync(
urls: list[str], httpc_params: dict = DEFAULT_HTTPC_PARAMS
) -> list[requests.Response]:
"""
Perform a GET http call sync.
:param urls: the url list where fetch the content
:type urls: list[str]
:param httpc_params: parameters to perform http requests.
:type httpc_params: dict
:raises HttpError: if the response status code is not 200 or a connection error occurs
:returns: the list of responses
:rtype: list[requests.Response]
"""
_conf = {
"verify": httpc_params["connection"]["ssl"],
"timeout": httpc_params["session"]["timeout"],
}
if "headers" in httpc_params:
_conf["headers"] = httpc_params["headers"]
try:
res = []
for url in urls:
if not _conf["verify"]:
logging.warning(
f"SSL certificate verification is disabled for HTTP request to: {url}"
)
logging.debug(f"Performing HTTP GET request to: {url}")
response : requests.Response = requests.get(url, **_conf) # nosec B113
res.append(response) # nosec - B113
except requests.exceptions.ConnectionError as e:
raise HttpError(f"Connection error: {e}")
for r in res:
if r.status_code != 200:
raise HttpError(f"HTTP error: {r.status_code} -- {r.reason}")
return res
[docs]
async def http_get_async(
urls, httpc_params: dict = DEFAULT_HTTPC_PARAMS
) -> list[requests.Response]:
"""
Perform a GET http call async.
:param session: a dict representing the current session
:type session: dict
:param urls: the url list where fetch the content
:type urls: list[str]
:param httpc_params: parameters to perform http requests.
:type httpc_params: dict
:raises HttpError: if the response status code is not 200 or a connection error occurs
:returns: the list of responses
:rtype: list[requests.Response]
"""
if not isinstance(httpc_params["session"]["timeout"], aiohttp.ClientTimeout):
httpc_params["session"]["timeout"] = aiohttp.ClientTimeout(
total=httpc_params["session"]["timeout"]
)
async with aiohttp.ClientSession(**httpc_params.get("session", {})) as session:
text = await fetch_all(session, urls, httpc_params)
return text
[docs]
async def fetch(
session: aiohttp.ClientSession, url: str, httpc_params: dict = DEFAULT_HTTPC_PARAMS
) -> aiohttp.ClientResponse:
"""
Fetches the content of a URL.
:param session: a dict representing the current session
:type session: dict
:param url: the url where fetch the content
:type url: str
:param httpc_params: parameters to perform http requests.
:type httpc_params: dict
:returns: the client response
:rtype: aiohttp.ClientResponse
"""
connection_params = httpc_params.get("connection", {})
if connection_params.get("ssl") is False:
logging.warning(f"SSL certificate verification is disabled for HTTP request to: {url}")
async with session.get(url, **connection_params) as response:
if response.status != 200: # pragma: no cover
response.raise_for_status()
return response
[docs]
async def fetch_all(
session: aiohttp.ClientSession,
urls: list[str],
httpc_params: dict = DEFAULT_HTTPC_PARAMS,
) -> list[requests.Response]:
"""
Fetches the content of a list of URL.
:param session: a dict representing the current session
:type session: dict
:param urls: the url list where fetch the content
:type urls: list[str]
:param httpc_params: parameters to perform http requests.
:type httpc_params: dict
:raises HttpError: if the response status code is not 200 or a connection error occurs
:returns: the list of responses in string format
:rtype: list[str]
"""
tasks = []
for url in urls:
task = asyncio.create_task(fetch(session, url, httpc_params))
tasks.append(task)
try:
results: list[requests.Response] = await asyncio.gather(*tasks)
except aiohttp.ClientConnectorError as e:
raise HttpError(f"Connection error: {e}")
for r in results:
if r.status_code != 200:
raise HttpError(f"HTTP error: {r.status_code} -- {r.reason}")
return results