From 99dd71989da538dd834331e32f68f8b3696ba605 Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Wed, 25 Feb 2026 00:10:00 +0100 Subject: [PATCH 1/7] debuger _geturlproductslist et request -erreur 403 --- main.py | 5 +++++ scraper.py | 21 +++++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index 59d1b2c..9bebe0e 100755 --- a/main.py +++ b/main.py @@ -12,6 +12,11 @@ def main() -> None: path: str = normpath(join(getcwd(), argv[1])) db: DataFrame = read_csv(path) print(db.all()) + print(db.head()) + print(db.info()) + print("\nnombre de valeurs manquantes pour chaque colonne :") + print(db.isna().sum()) + if __name__ == "__main__": try: diff --git a/scraper.py b/scraper.py index 736da5f..e951af1 100755 --- a/scraper.py +++ b/scraper.py @@ -151,10 +151,7 @@ class _ScraperData: parker = self.parker() robinson = self.robinson() suckling = self.suckling() - try: - prix = self.prix() - except ValueError: - prix = None + prix = self.prix() return f"{appellation},{parker},{robinson},{suckling},{prix}" @@ -194,7 +191,10 @@ class Scraper: HTTPError: Si le serveur renvoie un code d'erreur (4xx, 5xx). """ target_url: str = self._url + subdir.lstrip("/") + print(f"[DEBUG] GET {target_url}") response: Response = self._session.get(url=target_url, timeout=10) + print(f"[DEBUG] status={response.status_code} len={len(response.text)}") + print(f"[DEBUG] head={response.text[:120].replace('\\n',' ')}") response.raise_for_status() return response @@ -307,15 +307,20 @@ class Scraper: data: dict[str, object] = self.getjsondata(subdir).getdata() for element in ["initialReduxState", "categ", "content"]: - data: dict[str, object] = cast(dict[str, object], data.get(element)) - if not isinstance(data, dict): + nxt = data.get(element) + print("DEBUG key", element, "->", type(nxt)) + if not isinstance(nxt, dict): + print("DEBUG structure manquante, stop sur", element) return None + data = nxt - products: list[str] = cast(list[str], data.get("products")) + products = data.get("products") + print("DEBUG products type:", type(products), "len:", 0 if not isinstance(products, list) else len(products)) if isinstance(products, list): return products - except (JSONDecodeError, HTTPError): + except (JSONDecodeError, HTTPError) as e: + print(f"DEBUG HTTP/JSON error sur {subdir}: {type(e).__name__} {e}") return None def getvins(self, subdir: str, filename: str): From 73c622108007a14670a594f8505e75e6c40d310f Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Wed, 25 Feb 2026 02:48:55 +0100 Subject: [PATCH 2/7] ajout de la reprise automatique du scraping dans getvins --- scraper.py | 115 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 85 insertions(+), 30 deletions(-) diff --git a/scraper.py b/scraper.py index e951af1..43f7bb3 100755 --- a/scraper.py +++ b/scraper.py @@ -3,9 +3,12 @@ from sys import argv from typing import cast from requests import HTTPError, Response, Session +from requests.exceptions import Timeout, ConnectionError +import time from bs4 import BeautifulSoup, Tag from collections import OrderedDict from json import JSONDecodeError, loads +from pathlib import Path class _ScraperData: @@ -171,6 +174,12 @@ class Scraper: # Très utile pour éviter de renvoyer toujours les mêmes handshake # TCP et d'avoir toujours une connexion constante avec le server self._session: Session = Session() + self._session.headers.update({ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/122.0.0.0 Safari/537.36", + "Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8", + }) # Système de cache pour éviter de solliciter le serveur inutilement self._latest_request: tuple[(str, Response)] | None = None self._latest_soups: OrderedDict[str, BeautifulSoup] = OrderedDict[ @@ -191,12 +200,20 @@ class Scraper: HTTPError: Si le serveur renvoie un code d'erreur (4xx, 5xx). """ target_url: str = self._url + subdir.lstrip("/") - print(f"[DEBUG] GET {target_url}") - response: Response = self._session.get(url=target_url, timeout=10) - print(f"[DEBUG] status={response.status_code} len={len(response.text)}") - print(f"[DEBUG] head={response.text[:120].replace('\\n',' ')}") - response.raise_for_status() - return response + + last_exc: Exception | None = None + for attempt in range(1, 4): + try: + response: Response = self._session.get(url=target_url, timeout=30) + response.raise_for_status() + return response + except (Timeout, ConnectionError) as e: + last_exc = e + print(f"Timeout/ConnectionError ({attempt}/3) sur {target_url}: {e}") + time.sleep(2 * attempt) # 2s, 4s, 6s + + # après 3 essais, on abandonne + raise last_exc if last_exc else RuntimeError("Request failed") def getresponse(self, subdir: str = "", use_cache: bool = True) -> Response: """ @@ -307,22 +324,38 @@ class Scraper: data: dict[str, object] = self.getjsondata(subdir).getdata() for element in ["initialReduxState", "categ", "content"]: - nxt = data.get(element) - print("DEBUG key", element, "->", type(nxt)) - if not isinstance(nxt, dict): - print("DEBUG structure manquante, stop sur", element) + data: dict[str, object] = cast(dict[str, object], data.get(element)) + if not isinstance(data, dict): return None - data = nxt - products = data.get("products") - print("DEBUG products type:", type(products), "len:", 0 if not isinstance(products, list) else len(products)) + products: list[str] = cast(list[str], data.get("products")) if isinstance(products, list): return products - except (JSONDecodeError, HTTPError) as e: - print(f"DEBUG HTTP/JSON error sur {subdir}: {type(e).__name__} {e}") + except (JSONDecodeError, HTTPError): return None + def _save_progress(self, page: int, i: int, last_link: str) -> None: + Path("progress.txt").write_text(f"{page},{i},{last_link}", encoding="utf-8") + + + def _load_progress(self) -> tuple[int, int, str | None]: + p = Path("progress.txt") + if not p.exists(): + return (1, 0, None) + + try: + parts = p.read_text(encoding="utf-8").strip().split(",", 2) + + page = int(parts[0]) + i = int(parts[1]) + + last_link = parts[2] if len(parts) == 3 and parts[2] != "" else None + return (page, i, last_link) + + except Exception: + return (1, 0, None) + def getvins(self, subdir: str, filename: str): """_summary_ @@ -330,11 +363,17 @@ class Scraper: subdir (str): _description_ filename (str): _description_ """ - with open(filename, "a") as f: + start_page, start_i, last_link = self._load_progress() + print(f"__INFO__ Reprise à page={start_page}, index={start_i}, last_link={last_link}") + + with open(filename, "a", encoding="utf-8") as f: cache: set[str] = set[str]() - page = 0 - _ = f.write("Appellation,Robert,Robinson,Suckling,Prix\n") - + + if f.tell() == 0: + _ = f.write("Appellation,Robert,Robinson,Suckling,Prix\n") + + page = start_page - 1 + while True: page += 1 products_list = self._geturlproductslist(f"{subdir}?page={page}") @@ -343,24 +382,40 @@ class Scraper: break products_list_length = len(products_list) - for i, product in enumerate(products_list): + start_at = start_i if page == start_page else 0 + + for i in range(start_at, products_list_length): + product = products_list[i] if not isinstance(product, dict): continue link = product.get("seoKeyword") + if not link: + continue + + # pour eviter les doublons : + if (page == start_page) and (last_link is not None) and (link == last_link): + self._save_progress(page, + 1, link) + continue + + self._save_progress(page, i + 1, link) + + if link in cache: + continue + + try: + infos = self.getjsondata(link).informations() + _ = f.write(infos + "\n") + print(f"page: {page} | {i + 1}/{products_list_length} {link}") + cache.add(link) + + except (JSONDecodeError, HTTPError) as e: + print(f"Erreur sur le produit {link}: {e}") - if link and link not in cache: - try: - infos = self.getjsondata(link).informations() - _ = f.write(infos + "\n") - print( - f"page: {page} | {i + 1}/{products_list_length} {link}" - ) - cache.add(link) - except (JSONDecodeError, HTTPError) as e: - print(f"Erreur sur le produit {link}: {e}") f.flush() + Path("progress.txt").unlink(missing_ok=True) + def main() -> None: if len(argv) != 2: From f31de22693602ee768c9f4ee4942bd5a9b16cea0 Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Wed, 25 Feb 2026 03:49:36 +0100 Subject: [PATCH 3/7] Q9 suppression les lignes sans appellation --- main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 9bebe0e..5747f74 100755 --- a/main.py +++ b/main.py @@ -12,11 +12,12 @@ def main() -> None: path: str = normpath(join(getcwd(), argv[1])) db: DataFrame = read_csv(path) print(db.all()) - print(db.head()) print(db.info()) print("\nnombre de valeurs manquantes pour chaque colonne :") print(db.isna().sum()) - + db = db.dropna(subset=["Appellation"]) + db.to_csv("donnee_clean.csv", index=False) + print(db.isna().sum()) if __name__ == "__main__": try: From 5afb6e38feeab576a6875dbe3df288d03389b51f Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Thu, 26 Feb 2026 21:11:43 +0100 Subject: [PATCH 4/7] ajout : moyennes des notes par appellation --- cleaning.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++ main.py | 54 +++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 cleaning.py diff --git a/cleaning.py b/cleaning.py new file mode 100644 index 0000000..b7c66a4 --- /dev/null +++ b/cleaning.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +from pandas import DataFrame, to_numeric + + +def display_info(df: DataFrame) -> None: + print(df.all()) + print(df.info()) + print("\nNombre de valeurs manquantes par colonne :") + print(df.isna().sum()) + + +def drop_empty_appellation(df: DataFrame) -> DataFrame: + + return df.dropna(subset=["Appellation"]) + + +def mean_score(df: DataFrame, col: str) -> DataFrame: + """ + Calcule la moyenne d'une colonne de score par appellation. + - Convertit les valeurs en numériques, en remplaçant les non-convertibles par NaN + - Calcule la moyenne par appellation + - Remplace les NaN résultants par 0 + + """ + tmp = df[["Appellation", col]].copy() + + tmp[col] = to_numeric(tmp[col], errors="coerce") + + # moyenne par appellation + means = tmp.groupby("Appellation", as_index=False)[col].mean() + + means[col] = means[col].fillna(0) + + means = means.rename(columns={col: f"mean_{col}"}) + + return means + + +def mean_robert(df: DataFrame) -> DataFrame: + return mean_score(df, "Robert") + + +def mean_robinson(df: DataFrame) -> DataFrame: + return mean_score(df, "Robinson") + + +def mean_suckling(df: DataFrame) -> DataFrame: + return mean_score(df, "Suckling") \ No newline at end of file diff --git a/main.py b/main.py index 5747f74..a87f052 100755 --- a/main.py +++ b/main.py @@ -5,19 +5,53 @@ from os.path import normpath, join from sys import argv from pandas import read_csv, DataFrame +from cleaning import (display_info, + drop_empty_appellation, + mean_robert, + mean_robinson, + mean_suckling) + + +def load_csv(filename: str) -> DataFrame: + path: str = normpath(join(getcwd(), filename)) + return read_csv(path) + + +def save_csv(df: DataFrame, out_filename: str) -> None: + df.to_csv(out_filename, index=False) + + def main() -> None: if len(argv) != 2: - raise ValueError(f"{argv[0]} ") + raise ValueError(f"Usage: {argv[0]} ") + + df = load_csv(argv[1]) + + print("=== Avant nettoyage ===") + display_info(df) + + df = drop_empty_appellation(df) + save_csv(df, "donnee_clean.csv") + + print("\n=== Après nettoyage d'appellations manquantes ===") + display_info(df) + + #la moyenne des notes des vins pour chaque appellation + robert_means = mean_robert(df) + save_csv(robert_means, "mean_robert_by_appellation.csv") + print("\n=== moyenne Robert par appellation ===") + print(robert_means.head(10)) + + robinson_means = mean_robinson(df) + save_csv(robinson_means, "mean_robinson_by_appellation.csv") + print("\n===: moyennes Robinson par appellation ===") + print(robinson_means.head(10)) + + suckling_means = mean_suckling(df) + save_csv(suckling_means, "mean_suckling_by_appellation.csv") + print("\n===: moyennes Suckling par appellation ===") + print(suckling_means.head(10)) - path: str = normpath(join(getcwd(), argv[1])) - db: DataFrame = read_csv(path) - print(db.all()) - print(db.info()) - print("\nnombre de valeurs manquantes pour chaque colonne :") - print(db.isna().sum()) - db = db.dropna(subset=["Appellation"]) - db.to_csv("donnee_clean.csv", index=False) - print(db.isna().sum()) if __name__ == "__main__": try: From b0eb5df07ef1bb3fe0a63740742b9288069d56e8 Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Tue, 3 Mar 2026 03:18:35 +0100 Subject: [PATCH 5/7] ajout : remplac les notes manquantes par la moyenne de l'appellation --- cleaning.py | 32 ++++++++++++++++++++++++++++++-- main.py | 14 ++++++++++---- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/cleaning.py b/cleaning.py index b7c66a4..563c9e6 100644 --- a/cleaning.py +++ b/cleaning.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 from pandas import DataFrame, to_numeric +import pandas as pd + +SCORE_COLS = ["Robert", "Robinson", "Suckling"] def display_info(df: DataFrame) -> None: - print(df.all()) + df.describe() print(df.info()) print("\nNombre de valeurs manquantes par colonne :") print(df.isna().sum()) @@ -45,4 +48,29 @@ def mean_robinson(df: DataFrame) -> DataFrame: def mean_suckling(df: DataFrame) -> DataFrame: - return mean_score(df, "Suckling") \ No newline at end of file + return mean_score(df, "Suckling") + + +def fill_missing_scores(df: DataFrame) -> DataFrame: + """ + Remplacer les notes manquantes par la moyenne + des vins de la même appellation. + """ + df_copy = df.copy() + df_copy["Appellation"] = df_copy["Appellation"].astype(str).str.strip() + + for score in SCORE_COLS: + df_copy[score] = to_numeric(df_copy[score], errors="coerce") + + temp_cols: list[str] = [] + + for score in SCORE_COLS: + mean_df = mean_score(df_copy, score) + mean_name = f"mean_{score}" + temp_cols.append(mean_name) + + df_copy = df_copy.merge(mean_df, on="Appellation", how="left") + df_copy[score] = df_copy[score].fillna(df_copy[mean_name]) + + df_copy = df_copy.drop(columns=temp_cols) + return df_copy diff --git a/main.py b/main.py index a87f052..4d2e768 100755 --- a/main.py +++ b/main.py @@ -9,7 +9,9 @@ from cleaning import (display_info, drop_empty_appellation, mean_robert, mean_robinson, - mean_suckling) + mean_suckling, + fill_missing_scores, + encode_appellation) def load_csv(filename: str) -> DataFrame: @@ -44,14 +46,18 @@ def main() -> None: robinson_means = mean_robinson(df) save_csv(robinson_means, "mean_robinson_by_appellation.csv") - print("\n===: moyennes Robinson par appellation ===") + print("\n=== moyennes Robinson par appellation ===") print(robinson_means.head(10)) suckling_means = mean_suckling(df) save_csv(suckling_means, "mean_suckling_by_appellation.csv") - print("\n===: moyennes Suckling par appellation ===") + print("\n=== moyennes Suckling par appellation ===") print(suckling_means.head(10)) - + + df_missing_scores = fill_missing_scores(df) + save_csv(df_missing_scores, "donnee_filled.csv") + print("\n=== Après remplissage des notes manquantes ===") + display_info(df_missing_scores) if __name__ == "__main__": try: From 06097c257ee9a20a77a32e44bc781ce771012bf0 Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Tue, 3 Mar 2026 03:26:58 +0100 Subject: [PATCH 6/7] ajout : remplacer appellation par les colonnes indicatrices --- cleaning.py | 15 +++++++++++++++ main.py | 8 +++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cleaning.py b/cleaning.py index 563c9e6..efc1054 100644 --- a/cleaning.py +++ b/cleaning.py @@ -74,3 +74,18 @@ def fill_missing_scores(df: DataFrame) -> DataFrame: df_copy = df_copy.drop(columns=temp_cols) return df_copy + + +def encode_appellation(df: DataFrame, column: str = "Appellation") -> DataFrame: + """ + Remplace la colonne 'Appellation' par des colonnes indicatrices + """ + df_copy = df.copy() + + appellations = df_copy[column].astype(str).str.strip() + + appellation_dummies = pd.get_dummies(appellations) + + df_copy = df_copy.drop(columns=[column]) + + return df_copy.join(appellation_dummies) diff --git a/main.py b/main.py index 4d2e768..b59e373 100755 --- a/main.py +++ b/main.py @@ -57,7 +57,13 @@ def main() -> None: df_missing_scores = fill_missing_scores(df) save_csv(df_missing_scores, "donnee_filled.csv") print("\n=== Après remplissage des notes manquantes ===") - display_info(df_missing_scores) + display_info(df_missing_scores) + + df_ready = encode_appellation(df_missing_scores) + save_csv(df_ready, "donnee_ready.csv") + print("\n=== Après remplacer la colonne 'Appellation' par des colonnes indicatrices ===") + display_info(df_ready) + print(df_ready.filter(like="App_").any().head()) if __name__ == "__main__": try: From cefdb94dd5e8f26bb3c565ecb8e67cdbd9a7467b Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Tue, 3 Mar 2026 04:18:30 +0100 Subject: [PATCH 7/7] ajout : aout des tests test_cleaning.py --- cleaning.py | 23 ++++++++++++++--- main.py | 26 +++++++------------- test_cleaning.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 21 deletions(-) create mode 100644 test_cleaning.py diff --git a/cleaning.py b/cleaning.py index efc1054..fcaabdb 100644 --- a/cleaning.py +++ b/cleaning.py @@ -5,12 +5,27 @@ import pandas as pd SCORE_COLS = ["Robert", "Robinson", "Suckling"] -def display_info(df: DataFrame) -> None: - df.describe() - print(df.info()) - print("\nNombre de valeurs manquantes par colonne :") +def display_info(df: DataFrame, name: str = "DataFrame") -> None: + """ + Affiche un résumé du DataFrame + -la taille + -types des colonnes + -valeurs manquantes + -statistiques numériques + """ + print(f"\n===== {name} =====") + + print(f"Shape : {df.shape[0]} lignes × {df.shape[1]} colonnes") + + print("\nTypes des colonnes :") + print(df.dtypes) + + print("\nValeurs manquantes :") print(df.isna().sum()) + print("\nStatistiques numériques :") + print(df.describe().round(2)) + def drop_empty_appellation(df: DataFrame) -> DataFrame: diff --git a/main.py b/main.py index b59e373..65cbd62 100755 --- a/main.py +++ b/main.py @@ -29,41 +29,33 @@ def main() -> None: df = load_csv(argv[1]) - print("=== Avant nettoyage ===") - display_info(df) + display_info(df, "Avant le nettoyage") df = drop_empty_appellation(df) save_csv(df, "donnee_clean.csv") - - print("\n=== Après nettoyage d'appellations manquantes ===") - display_info(df) + display_info(df, "Après nettoyage d'appellations manquantes") #la moyenne des notes des vins pour chaque appellation robert_means = mean_robert(df) save_csv(robert_means, "mean_robert_by_appellation.csv") - print("\n=== moyenne Robert par appellation ===") - print(robert_means.head(10)) + display_info(robert_means, "Moyennes Robert par appellation") robinson_means = mean_robinson(df) save_csv(robinson_means, "mean_robinson_by_appellation.csv") - print("\n=== moyennes Robinson par appellation ===") - print(robinson_means.head(10)) - + display_info(robinson_means, "Moyennes Robinson par appellation") + suckling_means = mean_suckling(df) save_csv(suckling_means, "mean_suckling_by_appellation.csv") - print("\n=== moyennes Suckling par appellation ===") - print(suckling_means.head(10)) + display_info(suckling_means, "Moyennes Suckling par appellation") df_missing_scores = fill_missing_scores(df) save_csv(df_missing_scores, "donnee_filled.csv") - print("\n=== Après remplissage des notes manquantes ===") - display_info(df_missing_scores) + display_info(df_missing_scores, "Après remplissage des notes manquantes par la moyenne de l'appellation") df_ready = encode_appellation(df_missing_scores) save_csv(df_ready, "donnee_ready.csv") - print("\n=== Après remplacer la colonne 'Appellation' par des colonnes indicatrices ===") - display_info(df_ready) - print(df_ready.filter(like="App_").any().head()) + display_info(df_ready, "Après remplacer la colonne 'Appellation' par des colonnes indicatrices") + if __name__ == "__main__": try: diff --git a/test_cleaning.py b/test_cleaning.py new file mode 100644 index 0000000..d376ccf --- /dev/null +++ b/test_cleaning.py @@ -0,0 +1,64 @@ +import pandas as pd +import pytest +from pandas import DataFrame + +from cleaning import ( + SCORE_COLS, + drop_empty_appellation, + mean_score, + fill_missing_scores, + encode_appellation, +) + + +@pytest.fixture +def df_raw() -> DataFrame: + return pd.DataFrame({ + "Appellation": ["Pauillac", "Pauillac ", "Margaux", None, "Pomerol", "Pomerol"], + "Robert": ["95", None, "bad", 90, None, None], + "Robinson": [None, "93", 18, None, None, None], + "Suckling": [96, None, None, None, 91, None], + "Prix": ["10.0", "11.0", "20.0", "30.0", "40.0", "50.0"], + }) + + +def test_drop_empty_appellation(df_raw: DataFrame): + out = drop_empty_appellation(df_raw) + assert out["Appellation"].isna().sum() == 0 + assert len(out) == 5 + + +def test_mean_score_zero_when_no_scores(df_raw: DataFrame): + out = drop_empty_appellation(df_raw) + m = mean_score(out, "Robert") + assert list(m.columns) == ["Appellation", "mean_Robert"] + + # Pomerol n'a aucune note Robert => moyenne doit être 0 + pomerol_mean = m.loc[m["Appellation"].str.strip() == "Pomerol", "mean_Robert"].iloc[0] + assert pomerol_mean == 0 + + +def test_fill_missing_scores(df_raw: DataFrame): + out = drop_empty_appellation(df_raw) + filled = fill_missing_scores(out) + + # plus de NaN dans les colonnes de scores + for col in SCORE_COLS: + assert filled[col].isna().sum() == 0 + + assert filled.loc[1, "Robert"] == 95.0 + + # pas de colonnes temporaires mean_* + for col in SCORE_COLS: + assert f"mean_{col}" not in filled.columns + + +def test_encode_appellation(df_raw: DataFrame): + out = drop_empty_appellation(df_raw) + filled = fill_missing_scores(out) + encoded = encode_appellation(filled) + + # la colonne texte disparaît + assert "Appellation" not in encoded.columns + assert "Pauillac" in encoded.columns + assert encoded.loc[0, "Pauillac"] == 1 \ No newline at end of file