Source code for pyeudiw.tools.http

import asyncio

import aiohttp
import logging
import requests

from pyeudiw.tools.exceptions import HttpError
from pyeudiw.jwt.utils import decode_jwt_payload, is_jwt_format

DEFAULT_HTTPC_PARAMS = {
    "connection": {"ssl": False}, # FALSE FOR DEV ENVIRONMENT ONLY
    "session": {"timeout": 4},
}


[docs] def http_get( url: str, httpc_params: dict = DEFAULT_HTTPC_PARAMS, decode_output: bool = True ) -> str: """ Perform a GET http call. :param urls: the url list where fetch the content :type urls: list[str] :param httpc_params: parameters to perform http requests. :type httpc_params: dict :raises HttpError: if the response status code is not 200 or a connection error occurs :returns: the list of responses :rtype: list[requests.Response] | list[str] """ _conf = { "verify": httpc_params["connection"]["ssl"], "timeout": httpc_params["session"]["timeout"], } if not _conf["verify"]: logging.warning(f"SSL certificate verification is disabled for HTTP request to: {url}") try: # nosec B113: timeout is set via _conf["timeout"], which Bandit cannot detect through **_conf unpacking response = requests.get(url, **_conf) # nosec B113 except requests.exceptions.ConnectionError as e: raise HttpError(f"Connection error: {e}") if response.status_code != 200: raise HttpError(f"HTTP error: {response.status_code} -- {response.reason}") output = response.content if isinstance(output, bytes): output = output.decode("utf-8") if decode_output: if is_jwt_format(output): output = decode_jwt_payload(output) else: output = response.json() return output
[docs] def http_get_sync( urls: list[str], httpc_params: dict = DEFAULT_HTTPC_PARAMS ) -> list[requests.Response]: """ Perform a GET http call sync. :param urls: the url list where fetch the content :type urls: list[str] :param httpc_params: parameters to perform http requests. :type httpc_params: dict :raises HttpError: if the response status code is not 200 or a connection error occurs :returns: the list of responses :rtype: list[requests.Response] """ _conf = { "verify": httpc_params["connection"]["ssl"], "timeout": httpc_params["session"]["timeout"], } if "headers" in httpc_params: _conf["headers"] = httpc_params["headers"] try: res = [] for url in urls: if not _conf["verify"]: logging.warning( f"SSL certificate verification is disabled for HTTP request to: {url}" ) logging.debug(f"Performing HTTP GET request to: {url}") response : requests.Response = requests.get(url, **_conf) # nosec B113 res.append(response) # nosec - B113 except requests.exceptions.ConnectionError as e: raise HttpError(f"Connection error: {e}") for r in res: if r.status_code != 200: raise HttpError(f"HTTP error: {r.status_code} -- {r.reason}") return res
[docs] async def http_get_async( urls, httpc_params: dict = DEFAULT_HTTPC_PARAMS ) -> list[requests.Response]: """ Perform a GET http call async. :param session: a dict representing the current session :type session: dict :param urls: the url list where fetch the content :type urls: list[str] :param httpc_params: parameters to perform http requests. :type httpc_params: dict :raises HttpError: if the response status code is not 200 or a connection error occurs :returns: the list of responses :rtype: list[requests.Response] """ if not isinstance(httpc_params["session"]["timeout"], aiohttp.ClientTimeout): httpc_params["session"]["timeout"] = aiohttp.ClientTimeout( total=httpc_params["session"]["timeout"] ) async with aiohttp.ClientSession(**httpc_params.get("session", {})) as session: text = await fetch_all(session, urls, httpc_params) return text
[docs] async def fetch( session: aiohttp.ClientSession, url: str, httpc_params: dict = DEFAULT_HTTPC_PARAMS ) -> aiohttp.ClientResponse: """ Fetches the content of a URL. :param session: a dict representing the current session :type session: dict :param url: the url where fetch the content :type url: str :param httpc_params: parameters to perform http requests. :type httpc_params: dict :returns: the client response :rtype: aiohttp.ClientResponse """ connection_params = httpc_params.get("connection", {}) if connection_params.get("ssl") is False: logging.warning(f"SSL certificate verification is disabled for HTTP request to: {url}") async with session.get(url, **connection_params) as response: if response.status != 200: # pragma: no cover response.raise_for_status() return response
[docs] async def fetch_all( session: aiohttp.ClientSession, urls: list[str], httpc_params: dict = DEFAULT_HTTPC_PARAMS, ) -> list[requests.Response]: """ Fetches the content of a list of URL. :param session: a dict representing the current session :type session: dict :param urls: the url list where fetch the content :type urls: list[str] :param httpc_params: parameters to perform http requests. :type httpc_params: dict :raises HttpError: if the response status code is not 200 or a connection error occurs :returns: the list of responses in string format :rtype: list[str] """ tasks = [] for url in urls: task = asyncio.create_task(fetch(session, url, httpc_params)) tasks.append(task) try: results: list[requests.Response] = await asyncio.gather(*tasks) except aiohttp.ClientConnectorError as e: raise HttpError(f"Connection error: {e}") for r in results: if r.status_code != 200: raise HttpError(f"HTTP error: {r.status_code} -- {r.reason}") return results