mirror of
https://github.com/guezoloic/millesima_projetS6.git
synced 2026-03-28 19:13:42 +00:00
94 lines
3.7 KiB
Python
94 lines
3.7 KiB
Python
from sys import stderr
|
|
from typing import cast
|
|
from requests import Response, Session
|
|
from bs4 import BeautifulSoup, Tag
|
|
from json import JSONDecodeError, loads
|
|
|
|
|
|
class Scraper:
|
|
def __init__(self, subdir: str = "") -> None:
|
|
self._url: str = "https://www.millemisa.fr/"
|
|
self._session: Session = Session()
|
|
|
|
self._latest_request: tuple[(str, Response | None)] = ("", None)
|
|
|
|
def _request(self, subdir: str) -> Response:
|
|
target_url: str = self._url + subdir.lstrip("/")
|
|
response: Response = self._session.get(url=target_url, timeout=10)
|
|
response.raise_for_status()
|
|
return response
|
|
|
|
def getresponse(self, subdir: str = "") -> Response:
|
|
rq_subdir, rq_response = self._latest_request
|
|
|
|
if rq_response is None or subdir != rq_subdir:
|
|
request: Response = self._request(subdir)
|
|
self._latest_request = (subdir, request)
|
|
return request
|
|
|
|
return rq_response
|
|
|
|
def getsoup(self, subdir: str = "") -> BeautifulSoup:
|
|
markup: str = self.getresponse(subdir).text
|
|
return BeautifulSoup(markup, features="html.parser")
|
|
|
|
# def getjsondata(self, subdir: str = "", id: str = "__NEXT_DATA__") -> dict[str, object]:
|
|
# soup: BeautifulSoup = self.getsoup(subdir)
|
|
# # On s'assure que c'est bien un Tag pour avoir accès à .string
|
|
# script = soup.find("script", id=id)
|
|
|
|
# if isinstance(script, Tag) and script.string:
|
|
# try:
|
|
# # On commence avec le dictionnaire complet
|
|
# current_data: Any = loads(script.string)
|
|
|
|
# # Parcours de la structure imbriquée
|
|
# keys = ['props', 'pageProps', 'initialReduxState', 'product', 'content']
|
|
# for key in keys:
|
|
# if isinstance(current_data, dict) and key in current_data:
|
|
# current_data = current_data[key]
|
|
# else:
|
|
# # Si une clé manque, on lève une erreur explicite
|
|
# raise ValueError(f"Clé manquante dans le JSON : {key}")
|
|
|
|
# # On garantit à Pyright que le résultat final est un dictionnaire
|
|
# if isinstance(current_data, dict):
|
|
# return cast(dict[str, object], current_data)
|
|
|
|
# except (decoder.JSONDecodeError, ValueError) as e:
|
|
# print(f"Erreur lors de l'extraction JSON : {e}", file=stderr)
|
|
|
|
# return {}
|
|
|
|
def getjsondata(
|
|
self, subdir: str = "", id: str = "__NEXT_DATA__"
|
|
) -> dict[str, object]:
|
|
soup: BeautifulSoup = self.getsoup(subdir)
|
|
script: Tag | None = soup.find("script", id=id)
|
|
|
|
if isinstance(script, Tag) and script.string:
|
|
try:
|
|
current_data: object = loads(script.string)
|
|
# tout le chemin à parcourir pour arriver au données
|
|
# (plein d'information inutile)
|
|
keys: list[str] = [
|
|
"props",
|
|
"pageProps",
|
|
"initialReduxState",
|
|
"product",
|
|
"content",
|
|
]
|
|
for key in keys:
|
|
# si current_data est bien un dictionnaire et que la clé
|
|
# est bien dedans
|
|
if isinstance(current_data, dict) and key in current_data:
|
|
current_data = current_data[key]
|
|
else:
|
|
raise ValueError(f"Clé manquante dans le JSON : {key}")
|
|
|
|
if isinstance(current_data, dict):
|
|
return cast(dict[str, object], current_data)
|
|
|
|
except (JSONDecodeError, ValueError) as e:
|
|
print(f"Erreur lors de l'extraction JSON : {e}", file=stderr)
|
|
return {} |