From 99dd71989da538dd834331e32f68f8b3696ba605 Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Wed, 25 Feb 2026 00:10:00 +0100 Subject: [PATCH 01/12] debuger _geturlproductslist et request -erreur 403 --- main.py | 5 +++++ scraper.py | 21 +++++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index 59d1b2c..9bebe0e 100755 --- a/main.py +++ b/main.py @@ -12,6 +12,11 @@ def main() -> None: path: str = normpath(join(getcwd(), argv[1])) db: DataFrame = read_csv(path) print(db.all()) + print(db.head()) + print(db.info()) + print("\nnombre de valeurs manquantes pour chaque colonne :") + print(db.isna().sum()) + if __name__ == "__main__": try: diff --git a/scraper.py b/scraper.py index 736da5f..e951af1 100755 --- a/scraper.py +++ b/scraper.py @@ -151,10 +151,7 @@ class _ScraperData: parker = self.parker() robinson = self.robinson() suckling = self.suckling() - try: - prix = self.prix() - except ValueError: - prix = None + prix = self.prix() return f"{appellation},{parker},{robinson},{suckling},{prix}" @@ -194,7 +191,10 @@ class Scraper: HTTPError: Si le serveur renvoie un code d'erreur (4xx, 5xx). """ target_url: str = self._url + subdir.lstrip("/") + print(f"[DEBUG] GET {target_url}") response: Response = self._session.get(url=target_url, timeout=10) + print(f"[DEBUG] status={response.status_code} len={len(response.text)}") + print(f"[DEBUG] head={response.text[:120].replace('\\n',' ')}") response.raise_for_status() return response @@ -307,15 +307,20 @@ class Scraper: data: dict[str, object] = self.getjsondata(subdir).getdata() for element in ["initialReduxState", "categ", "content"]: - data: dict[str, object] = cast(dict[str, object], data.get(element)) - if not isinstance(data, dict): + nxt = data.get(element) + print("DEBUG key", element, "->", type(nxt)) + if not isinstance(nxt, dict): + print("DEBUG structure manquante, stop sur", element) return None + data = nxt - products: list[str] = cast(list[str], data.get("products")) + products = data.get("products") + print("DEBUG products type:", type(products), "len:", 0 if not isinstance(products, list) else len(products)) if isinstance(products, list): return products - except (JSONDecodeError, HTTPError): + except (JSONDecodeError, HTTPError) as e: + print(f"DEBUG HTTP/JSON error sur {subdir}: {type(e).__name__} {e}") return None def getvins(self, subdir: str, filename: str): From 73c622108007a14670a594f8505e75e6c40d310f Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Wed, 25 Feb 2026 02:48:55 +0100 Subject: [PATCH 02/12] ajout de la reprise automatique du scraping dans getvins --- scraper.py | 115 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 85 insertions(+), 30 deletions(-) diff --git a/scraper.py b/scraper.py index e951af1..43f7bb3 100755 --- a/scraper.py +++ b/scraper.py @@ -3,9 +3,12 @@ from sys import argv from typing import cast from requests import HTTPError, Response, Session +from requests.exceptions import Timeout, ConnectionError +import time from bs4 import BeautifulSoup, Tag from collections import OrderedDict from json import JSONDecodeError, loads +from pathlib import Path class _ScraperData: @@ -171,6 +174,12 @@ class Scraper: # Très utile pour éviter de renvoyer toujours les mêmes handshake # TCP et d'avoir toujours une connexion constante avec le server self._session: Session = Session() + self._session.headers.update({ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/122.0.0.0 Safari/537.36", + "Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8", + }) # Système de cache pour éviter de solliciter le serveur inutilement self._latest_request: tuple[(str, Response)] | None = None self._latest_soups: OrderedDict[str, BeautifulSoup] = OrderedDict[ @@ -191,12 +200,20 @@ class Scraper: HTTPError: Si le serveur renvoie un code d'erreur (4xx, 5xx). """ target_url: str = self._url + subdir.lstrip("/") - print(f"[DEBUG] GET {target_url}") - response: Response = self._session.get(url=target_url, timeout=10) - print(f"[DEBUG] status={response.status_code} len={len(response.text)}") - print(f"[DEBUG] head={response.text[:120].replace('\\n',' ')}") - response.raise_for_status() - return response + + last_exc: Exception | None = None + for attempt in range(1, 4): + try: + response: Response = self._session.get(url=target_url, timeout=30) + response.raise_for_status() + return response + except (Timeout, ConnectionError) as e: + last_exc = e + print(f"Timeout/ConnectionError ({attempt}/3) sur {target_url}: {e}") + time.sleep(2 * attempt) # 2s, 4s, 6s + + # après 3 essais, on abandonne + raise last_exc if last_exc else RuntimeError("Request failed") def getresponse(self, subdir: str = "", use_cache: bool = True) -> Response: """ @@ -307,22 +324,38 @@ class Scraper: data: dict[str, object] = self.getjsondata(subdir).getdata() for element in ["initialReduxState", "categ", "content"]: - nxt = data.get(element) - print("DEBUG key", element, "->", type(nxt)) - if not isinstance(nxt, dict): - print("DEBUG structure manquante, stop sur", element) + data: dict[str, object] = cast(dict[str, object], data.get(element)) + if not isinstance(data, dict): return None - data = nxt - products = data.get("products") - print("DEBUG products type:", type(products), "len:", 0 if not isinstance(products, list) else len(products)) + products: list[str] = cast(list[str], data.get("products")) if isinstance(products, list): return products - except (JSONDecodeError, HTTPError) as e: - print(f"DEBUG HTTP/JSON error sur {subdir}: {type(e).__name__} {e}") + except (JSONDecodeError, HTTPError): return None + def _save_progress(self, page: int, i: int, last_link: str) -> None: + Path("progress.txt").write_text(f"{page},{i},{last_link}", encoding="utf-8") + + + def _load_progress(self) -> tuple[int, int, str | None]: + p = Path("progress.txt") + if not p.exists(): + return (1, 0, None) + + try: + parts = p.read_text(encoding="utf-8").strip().split(",", 2) + + page = int(parts[0]) + i = int(parts[1]) + + last_link = parts[2] if len(parts) == 3 and parts[2] != "" else None + return (page, i, last_link) + + except Exception: + return (1, 0, None) + def getvins(self, subdir: str, filename: str): """_summary_ @@ -330,11 +363,17 @@ class Scraper: subdir (str): _description_ filename (str): _description_ """ - with open(filename, "a") as f: + start_page, start_i, last_link = self._load_progress() + print(f"__INFO__ Reprise à page={start_page}, index={start_i}, last_link={last_link}") + + with open(filename, "a", encoding="utf-8") as f: cache: set[str] = set[str]() - page = 0 - _ = f.write("Appellation,Robert,Robinson,Suckling,Prix\n") - + + if f.tell() == 0: + _ = f.write("Appellation,Robert,Robinson,Suckling,Prix\n") + + page = start_page - 1 + while True: page += 1 products_list = self._geturlproductslist(f"{subdir}?page={page}") @@ -343,24 +382,40 @@ class Scraper: break products_list_length = len(products_list) - for i, product in enumerate(products_list): + start_at = start_i if page == start_page else 0 + + for i in range(start_at, products_list_length): + product = products_list[i] if not isinstance(product, dict): continue link = product.get("seoKeyword") + if not link: + continue + + # pour eviter les doublons : + if (page == start_page) and (last_link is not None) and (link == last_link): + self._save_progress(page, + 1, link) + continue + + self._save_progress(page, i + 1, link) + + if link in cache: + continue + + try: + infos = self.getjsondata(link).informations() + _ = f.write(infos + "\n") + print(f"page: {page} | {i + 1}/{products_list_length} {link}") + cache.add(link) + + except (JSONDecodeError, HTTPError) as e: + print(f"Erreur sur le produit {link}: {e}") - if link and link not in cache: - try: - infos = self.getjsondata(link).informations() - _ = f.write(infos + "\n") - print( - f"page: {page} | {i + 1}/{products_list_length} {link}" - ) - cache.add(link) - except (JSONDecodeError, HTTPError) as e: - print(f"Erreur sur le produit {link}: {e}") f.flush() + Path("progress.txt").unlink(missing_ok=True) + def main() -> None: if len(argv) != 2: From f31de22693602ee768c9f4ee4942bd5a9b16cea0 Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Wed, 25 Feb 2026 03:49:36 +0100 Subject: [PATCH 03/12] Q9 suppression les lignes sans appellation --- main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 9bebe0e..5747f74 100755 --- a/main.py +++ b/main.py @@ -12,11 +12,12 @@ def main() -> None: path: str = normpath(join(getcwd(), argv[1])) db: DataFrame = read_csv(path) print(db.all()) - print(db.head()) print(db.info()) print("\nnombre de valeurs manquantes pour chaque colonne :") print(db.isna().sum()) - + db = db.dropna(subset=["Appellation"]) + db.to_csv("donnee_clean.csv", index=False) + print(db.isna().sum()) if __name__ == "__main__": try: From 5afb6e38feeab576a6875dbe3df288d03389b51f Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Thu, 26 Feb 2026 21:11:43 +0100 Subject: [PATCH 04/12] ajout : moyennes des notes par appellation --- cleaning.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++ main.py | 54 +++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 cleaning.py diff --git a/cleaning.py b/cleaning.py new file mode 100644 index 0000000..b7c66a4 --- /dev/null +++ b/cleaning.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +from pandas import DataFrame, to_numeric + + +def display_info(df: DataFrame) -> None: + print(df.all()) + print(df.info()) + print("\nNombre de valeurs manquantes par colonne :") + print(df.isna().sum()) + + +def drop_empty_appellation(df: DataFrame) -> DataFrame: + + return df.dropna(subset=["Appellation"]) + + +def mean_score(df: DataFrame, col: str) -> DataFrame: + """ + Calcule la moyenne d'une colonne de score par appellation. + - Convertit les valeurs en numériques, en remplaçant les non-convertibles par NaN + - Calcule la moyenne par appellation + - Remplace les NaN résultants par 0 + + """ + tmp = df[["Appellation", col]].copy() + + tmp[col] = to_numeric(tmp[col], errors="coerce") + + # moyenne par appellation + means = tmp.groupby("Appellation", as_index=False)[col].mean() + + means[col] = means[col].fillna(0) + + means = means.rename(columns={col: f"mean_{col}"}) + + return means + + +def mean_robert(df: DataFrame) -> DataFrame: + return mean_score(df, "Robert") + + +def mean_robinson(df: DataFrame) -> DataFrame: + return mean_score(df, "Robinson") + + +def mean_suckling(df: DataFrame) -> DataFrame: + return mean_score(df, "Suckling") \ No newline at end of file diff --git a/main.py b/main.py index 5747f74..a87f052 100755 --- a/main.py +++ b/main.py @@ -5,19 +5,53 @@ from os.path import normpath, join from sys import argv from pandas import read_csv, DataFrame +from cleaning import (display_info, + drop_empty_appellation, + mean_robert, + mean_robinson, + mean_suckling) + + +def load_csv(filename: str) -> DataFrame: + path: str = normpath(join(getcwd(), filename)) + return read_csv(path) + + +def save_csv(df: DataFrame, out_filename: str) -> None: + df.to_csv(out_filename, index=False) + + def main() -> None: if len(argv) != 2: - raise ValueError(f"{argv[0]} ") + raise ValueError(f"Usage: {argv[0]} ") + + df = load_csv(argv[1]) + + print("=== Avant nettoyage ===") + display_info(df) + + df = drop_empty_appellation(df) + save_csv(df, "donnee_clean.csv") + + print("\n=== Après nettoyage d'appellations manquantes ===") + display_info(df) + + #la moyenne des notes des vins pour chaque appellation + robert_means = mean_robert(df) + save_csv(robert_means, "mean_robert_by_appellation.csv") + print("\n=== moyenne Robert par appellation ===") + print(robert_means.head(10)) + + robinson_means = mean_robinson(df) + save_csv(robinson_means, "mean_robinson_by_appellation.csv") + print("\n===: moyennes Robinson par appellation ===") + print(robinson_means.head(10)) + + suckling_means = mean_suckling(df) + save_csv(suckling_means, "mean_suckling_by_appellation.csv") + print("\n===: moyennes Suckling par appellation ===") + print(suckling_means.head(10)) - path: str = normpath(join(getcwd(), argv[1])) - db: DataFrame = read_csv(path) - print(db.all()) - print(db.info()) - print("\nnombre de valeurs manquantes pour chaque colonne :") - print(db.isna().sum()) - db = db.dropna(subset=["Appellation"]) - db.to_csv("donnee_clean.csv", index=False) - print(db.isna().sum()) if __name__ == "__main__": try: From b0eb5df07ef1bb3fe0a63740742b9288069d56e8 Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Tue, 3 Mar 2026 03:18:35 +0100 Subject: [PATCH 05/12] ajout : remplac les notes manquantes par la moyenne de l'appellation --- cleaning.py | 32 ++++++++++++++++++++++++++++++-- main.py | 14 ++++++++++---- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/cleaning.py b/cleaning.py index b7c66a4..563c9e6 100644 --- a/cleaning.py +++ b/cleaning.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 from pandas import DataFrame, to_numeric +import pandas as pd + +SCORE_COLS = ["Robert", "Robinson", "Suckling"] def display_info(df: DataFrame) -> None: - print(df.all()) + df.describe() print(df.info()) print("\nNombre de valeurs manquantes par colonne :") print(df.isna().sum()) @@ -45,4 +48,29 @@ def mean_robinson(df: DataFrame) -> DataFrame: def mean_suckling(df: DataFrame) -> DataFrame: - return mean_score(df, "Suckling") \ No newline at end of file + return mean_score(df, "Suckling") + + +def fill_missing_scores(df: DataFrame) -> DataFrame: + """ + Remplacer les notes manquantes par la moyenne + des vins de la même appellation. + """ + df_copy = df.copy() + df_copy["Appellation"] = df_copy["Appellation"].astype(str).str.strip() + + for score in SCORE_COLS: + df_copy[score] = to_numeric(df_copy[score], errors="coerce") + + temp_cols: list[str] = [] + + for score in SCORE_COLS: + mean_df = mean_score(df_copy, score) + mean_name = f"mean_{score}" + temp_cols.append(mean_name) + + df_copy = df_copy.merge(mean_df, on="Appellation", how="left") + df_copy[score] = df_copy[score].fillna(df_copy[mean_name]) + + df_copy = df_copy.drop(columns=temp_cols) + return df_copy diff --git a/main.py b/main.py index a87f052..4d2e768 100755 --- a/main.py +++ b/main.py @@ -9,7 +9,9 @@ from cleaning import (display_info, drop_empty_appellation, mean_robert, mean_robinson, - mean_suckling) + mean_suckling, + fill_missing_scores, + encode_appellation) def load_csv(filename: str) -> DataFrame: @@ -44,14 +46,18 @@ def main() -> None: robinson_means = mean_robinson(df) save_csv(robinson_means, "mean_robinson_by_appellation.csv") - print("\n===: moyennes Robinson par appellation ===") + print("\n=== moyennes Robinson par appellation ===") print(robinson_means.head(10)) suckling_means = mean_suckling(df) save_csv(suckling_means, "mean_suckling_by_appellation.csv") - print("\n===: moyennes Suckling par appellation ===") + print("\n=== moyennes Suckling par appellation ===") print(suckling_means.head(10)) - + + df_missing_scores = fill_missing_scores(df) + save_csv(df_missing_scores, "donnee_filled.csv") + print("\n=== Après remplissage des notes manquantes ===") + display_info(df_missing_scores) if __name__ == "__main__": try: From 06097c257ee9a20a77a32e44bc781ce771012bf0 Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Tue, 3 Mar 2026 03:26:58 +0100 Subject: [PATCH 06/12] ajout : remplacer appellation par les colonnes indicatrices --- cleaning.py | 15 +++++++++++++++ main.py | 8 +++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cleaning.py b/cleaning.py index 563c9e6..efc1054 100644 --- a/cleaning.py +++ b/cleaning.py @@ -74,3 +74,18 @@ def fill_missing_scores(df: DataFrame) -> DataFrame: df_copy = df_copy.drop(columns=temp_cols) return df_copy + + +def encode_appellation(df: DataFrame, column: str = "Appellation") -> DataFrame: + """ + Remplace la colonne 'Appellation' par des colonnes indicatrices + """ + df_copy = df.copy() + + appellations = df_copy[column].astype(str).str.strip() + + appellation_dummies = pd.get_dummies(appellations) + + df_copy = df_copy.drop(columns=[column]) + + return df_copy.join(appellation_dummies) diff --git a/main.py b/main.py index 4d2e768..b59e373 100755 --- a/main.py +++ b/main.py @@ -57,7 +57,13 @@ def main() -> None: df_missing_scores = fill_missing_scores(df) save_csv(df_missing_scores, "donnee_filled.csv") print("\n=== Après remplissage des notes manquantes ===") - display_info(df_missing_scores) + display_info(df_missing_scores) + + df_ready = encode_appellation(df_missing_scores) + save_csv(df_ready, "donnee_ready.csv") + print("\n=== Après remplacer la colonne 'Appellation' par des colonnes indicatrices ===") + display_info(df_ready) + print(df_ready.filter(like="App_").any().head()) if __name__ == "__main__": try: From cefdb94dd5e8f26bb3c565ecb8e67cdbd9a7467b Mon Sep 17 00:00:00 2001 From: Chahrazad650 Date: Tue, 3 Mar 2026 04:18:30 +0100 Subject: [PATCH 07/12] ajout : aout des tests test_cleaning.py --- cleaning.py | 23 ++++++++++++++--- main.py | 26 +++++++------------- test_cleaning.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 21 deletions(-) create mode 100644 test_cleaning.py diff --git a/cleaning.py b/cleaning.py index efc1054..fcaabdb 100644 --- a/cleaning.py +++ b/cleaning.py @@ -5,12 +5,27 @@ import pandas as pd SCORE_COLS = ["Robert", "Robinson", "Suckling"] -def display_info(df: DataFrame) -> None: - df.describe() - print(df.info()) - print("\nNombre de valeurs manquantes par colonne :") +def display_info(df: DataFrame, name: str = "DataFrame") -> None: + """ + Affiche un résumé du DataFrame + -la taille + -types des colonnes + -valeurs manquantes + -statistiques numériques + """ + print(f"\n===== {name} =====") + + print(f"Shape : {df.shape[0]} lignes × {df.shape[1]} colonnes") + + print("\nTypes des colonnes :") + print(df.dtypes) + + print("\nValeurs manquantes :") print(df.isna().sum()) + print("\nStatistiques numériques :") + print(df.describe().round(2)) + def drop_empty_appellation(df: DataFrame) -> DataFrame: diff --git a/main.py b/main.py index b59e373..65cbd62 100755 --- a/main.py +++ b/main.py @@ -29,41 +29,33 @@ def main() -> None: df = load_csv(argv[1]) - print("=== Avant nettoyage ===") - display_info(df) + display_info(df, "Avant le nettoyage") df = drop_empty_appellation(df) save_csv(df, "donnee_clean.csv") - - print("\n=== Après nettoyage d'appellations manquantes ===") - display_info(df) + display_info(df, "Après nettoyage d'appellations manquantes") #la moyenne des notes des vins pour chaque appellation robert_means = mean_robert(df) save_csv(robert_means, "mean_robert_by_appellation.csv") - print("\n=== moyenne Robert par appellation ===") - print(robert_means.head(10)) + display_info(robert_means, "Moyennes Robert par appellation") robinson_means = mean_robinson(df) save_csv(robinson_means, "mean_robinson_by_appellation.csv") - print("\n=== moyennes Robinson par appellation ===") - print(robinson_means.head(10)) - + display_info(robinson_means, "Moyennes Robinson par appellation") + suckling_means = mean_suckling(df) save_csv(suckling_means, "mean_suckling_by_appellation.csv") - print("\n=== moyennes Suckling par appellation ===") - print(suckling_means.head(10)) + display_info(suckling_means, "Moyennes Suckling par appellation") df_missing_scores = fill_missing_scores(df) save_csv(df_missing_scores, "donnee_filled.csv") - print("\n=== Après remplissage des notes manquantes ===") - display_info(df_missing_scores) + display_info(df_missing_scores, "Après remplissage des notes manquantes par la moyenne de l'appellation") df_ready = encode_appellation(df_missing_scores) save_csv(df_ready, "donnee_ready.csv") - print("\n=== Après remplacer la colonne 'Appellation' par des colonnes indicatrices ===") - display_info(df_ready) - print(df_ready.filter(like="App_").any().head()) + display_info(df_ready, "Après remplacer la colonne 'Appellation' par des colonnes indicatrices") + if __name__ == "__main__": try: diff --git a/test_cleaning.py b/test_cleaning.py new file mode 100644 index 0000000..d376ccf --- /dev/null +++ b/test_cleaning.py @@ -0,0 +1,64 @@ +import pandas as pd +import pytest +from pandas import DataFrame + +from cleaning import ( + SCORE_COLS, + drop_empty_appellation, + mean_score, + fill_missing_scores, + encode_appellation, +) + + +@pytest.fixture +def df_raw() -> DataFrame: + return pd.DataFrame({ + "Appellation": ["Pauillac", "Pauillac ", "Margaux", None, "Pomerol", "Pomerol"], + "Robert": ["95", None, "bad", 90, None, None], + "Robinson": [None, "93", 18, None, None, None], + "Suckling": [96, None, None, None, 91, None], + "Prix": ["10.0", "11.0", "20.0", "30.0", "40.0", "50.0"], + }) + + +def test_drop_empty_appellation(df_raw: DataFrame): + out = drop_empty_appellation(df_raw) + assert out["Appellation"].isna().sum() == 0 + assert len(out) == 5 + + +def test_mean_score_zero_when_no_scores(df_raw: DataFrame): + out = drop_empty_appellation(df_raw) + m = mean_score(out, "Robert") + assert list(m.columns) == ["Appellation", "mean_Robert"] + + # Pomerol n'a aucune note Robert => moyenne doit être 0 + pomerol_mean = m.loc[m["Appellation"].str.strip() == "Pomerol", "mean_Robert"].iloc[0] + assert pomerol_mean == 0 + + +def test_fill_missing_scores(df_raw: DataFrame): + out = drop_empty_appellation(df_raw) + filled = fill_missing_scores(out) + + # plus de NaN dans les colonnes de scores + for col in SCORE_COLS: + assert filled[col].isna().sum() == 0 + + assert filled.loc[1, "Robert"] == 95.0 + + # pas de colonnes temporaires mean_* + for col in SCORE_COLS: + assert f"mean_{col}" not in filled.columns + + +def test_encode_appellation(df_raw: DataFrame): + out = drop_empty_appellation(df_raw) + filled = fill_missing_scores(out) + encoded = encode_appellation(filled) + + # la colonne texte disparaît + assert "Appellation" not in encoded.columns + assert "Pauillac" in encoded.columns + assert encoded.loc[0, "Pauillac"] == 1 \ No newline at end of file From 8047b06253871adfa5466601114da87957a19948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20GUEZO?= Date: Thu, 5 Mar 2026 20:00:23 +0100 Subject: [PATCH 08/12] fix: sauvegarde toute les 5 pages --- src/scraper.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/scraper.py b/src/scraper.py index 5f45d30..bc599a6 100755 --- a/src/scraper.py +++ b/src/scraper.py @@ -384,8 +384,7 @@ class Scraper: list[dict[str, Any]], data.get("products") ) - if isinstance(products, list): - return products + return products except (JSONDecodeError, HTTPError): return None @@ -461,12 +460,18 @@ class Scraper: products_list, bar_format=custom_format ) for product in pbar: - keyword = product.get("seoKeyword", "Inconnu")[:40] + keyword: str = cast( + str, product.get("seoKeyword", "Inconnu")[:40] + ) pbar.set_description( f"Page: {page:<3} | Product: {keyword:<40}" ) self._writevins(cache, product, f) page += 1 + # va créer un fichier au début et l'override + # tout les 5 pages au cas où SIGHUP ou autre + if page % 5 == 0 and not reset: + savestate((page, cache)) except (Exception, HTTPError, KeyboardInterrupt, JSONDecodeError): if not reset: savestate((page, cache)) From 69b8b4ce1f3374de48b2bf89bd1137e49144802c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20GUEZO?= Date: Thu, 5 Mar 2026 22:06:00 +0100 Subject: [PATCH 09/12] ajout: restructuration du code --- pyproject.toml | 7 ++++++- src/cleaning.py | 15 ++++++--------- src/main.py | 8 +------- tests/test_cleaning.py | 0 tests/test_scraper.py | 0 5 files changed, 13 insertions(+), 17 deletions(-) mode change 100644 => 100755 tests/test_cleaning.py mode change 100644 => 100755 tests/test_scraper.py diff --git a/pyproject.toml b/pyproject.toml index 6d14d59..e638cbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,12 @@ [project] name = "projet-millesima-s6" version = "0.1.0" -dependencies = ["requests==2.32.5", "beautifulsoup4==4.14.3", "pandas==2.3.3", "tqdm==4.67.3"] +dependencies = [ + "requests==2.32.5", + "beautifulsoup4==4.14.3", + "pandas==2.3.3", + "tqdm==4.67.3", +] [project.optional-dependencies] test = ["pytest==8.4.2", "requests-mock==1.12.1", "flake8==7.3.0"] diff --git a/src/cleaning.py b/src/cleaning.py index fcaabdb..1f3f788 100644 --- a/src/cleaning.py +++ b/src/cleaning.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -from pandas import DataFrame, to_numeric -import pandas as pd +from pandas import DataFrame, to_numeric, get_dummies SCORE_COLS = ["Robert", "Robinson", "Suckling"] @@ -37,7 +36,7 @@ def mean_score(df: DataFrame, col: str) -> DataFrame: Calcule la moyenne d'une colonne de score par appellation. - Convertit les valeurs en numériques, en remplaçant les non-convertibles par NaN - Calcule la moyenne par appellation - - Remplace les NaN résultants par 0 + - Remplace les NaN résultants par 0 """ tmp = df[["Appellation", col]].copy() @@ -46,12 +45,10 @@ def mean_score(df: DataFrame, col: str) -> DataFrame: # moyenne par appellation means = tmp.groupby("Appellation", as_index=False)[col].mean() - + means[col] = means[col].fillna(0) - + means = means.rename(columns={col: f"mean_{col}"}) - - return means def mean_robert(df: DataFrame) -> DataFrame: @@ -96,10 +93,10 @@ def encode_appellation(df: DataFrame, column: str = "Appellation") -> DataFrame: Remplace la colonne 'Appellation' par des colonnes indicatrices """ df_copy = df.copy() - + appellations = df_copy[column].astype(str).str.strip() - appellation_dummies = pd.get_dummies(appellations) + appellation_dummies = get_dummies(appellations) df_copy = df_copy.drop(columns=[column]) diff --git a/src/main.py b/src/main.py index 65cbd62..512fe20 100755 --- a/src/main.py +++ b/src/main.py @@ -5,13 +5,7 @@ from os.path import normpath, join from sys import argv from pandas import read_csv, DataFrame -from cleaning import (display_info, - drop_empty_appellation, - mean_robert, - mean_robinson, - mean_suckling, - fill_missing_scores, - encode_appellation) +from cleaning import * def load_csv(filename: str) -> DataFrame: diff --git a/tests/test_cleaning.py b/tests/test_cleaning.py old mode 100644 new mode 100755 diff --git a/tests/test_scraper.py b/tests/test_scraper.py old mode 100644 new mode 100755 From acf4ddd881f9b0b027ad3a951a20102d85be32c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20GUEZO?= Date: Fri, 6 Mar 2026 17:56:07 +0100 Subject: [PATCH 10/12] ajout: restructuration de la cleaning --- src/cleaning.py | 160 +++++++++++++++++++---------------------- tests/test_cleaning.py | 94 ++++++++++++------------ 2 files changed, 121 insertions(+), 133 deletions(-) mode change 100644 => 100755 src/cleaning.py diff --git a/src/cleaning.py b/src/cleaning.py old mode 100644 new mode 100755 index 1f3f788..781896b --- a/src/cleaning.py +++ b/src/cleaning.py @@ -1,103 +1,87 @@ #!/usr/bin/env python3 -from pandas import DataFrame, to_numeric, get_dummies -SCORE_COLS = ["Robert", "Robinson", "Suckling"] +from typing import cast, override +from pandas import DataFrame, read_csv, to_numeric, get_dummies -def display_info(df: DataFrame, name: str = "DataFrame") -> None: - """ - Affiche un résumé du DataFrame - -la taille - -types des colonnes - -valeurs manquantes - -statistiques numériques - """ - print(f"\n===== {name} =====") +class Cleaning: + def __init__(self, filename) -> None: + self._vins: DataFrame = read_csv(filename) + # + self.SCORE_COLS: list[str] = [ + c for c in self._vins.columns if c not in ["Appellation", "Prix"] + ] + # + for col in self.SCORE_COLS: + self._vins[col] = to_numeric(self._vins[col], errors="coerce") - print(f"Shape : {df.shape[0]} lignes × {df.shape[1]} colonnes") + def getVins(self) -> DataFrame: + return self._vins.copy(deep=True) - print("\nTypes des colonnes :") - print(df.dtypes) + @override + def __str__(self) -> str: + """ + Affiche un résumé du DataFrame + - la taille + - types des colonnes + - valeurs manquantes + - statistiques numériques + """ + return ( + f"Shape : {self._vins.shape[0]} lignes x {self._vins.shape[1]} colonnes\n\n" + f"Types des colonnes :\n{self._vins.dtypes}\n\n" + f"Valeurs manquantes :\n{self._vins.isna().sum()}\n\n" + f"Statistiques numériques :\n{self._vins.describe().round(2)}\n\n" + ) - print("\nValeurs manquantes :") - print(df.isna().sum()) + def drop_empty_appellation(self) -> Cleaning: + self._vins = self._vins.dropna(subset=["Appellation"]) + return self - print("\nStatistiques numériques :") - print(df.describe().round(2)) + def _mean_score(self, col: str) -> DataFrame: + """ + Calcule la moyenne d'une colonne de score par appellation. + - Convertit les valeurs en numériques, en remplaçant les non-convertibles par NaN + - Calcule la moyenne par appellation + - Remplace les NaN résultants par 0 + """ + means = self._vins.groupby("Appellation", as_index=False)[col].mean() + means = means.rename( + columns={col: f"mean_{col}"} + ) # pyright: ignore[reportCallIssue] + return cast(DataFrame, means.fillna(0)) -def drop_empty_appellation(df: DataFrame) -> DataFrame: + def _mean_robert(self) -> DataFrame: + return self._mean_score("Robert") - return df.dropna(subset=["Appellation"]) + def _mean_robinson(self) -> DataFrame: + return self._mean_score("Robinson") + def _mean_suckling(self) -> DataFrame: + return self._mean_score("Suckling") -def mean_score(df: DataFrame, col: str) -> DataFrame: - """ - Calcule la moyenne d'une colonne de score par appellation. - - Convertit les valeurs en numériques, en remplaçant les non-convertibles par NaN - - Calcule la moyenne par appellation - - Remplace les NaN résultants par 0 + def fill_missing_scores(self) -> Cleaning: + """ + Remplacer les notes manquantes par la moyenne + des vins de la même appellation. + """ + for element in self.SCORE_COLS: + means = self._mean_score(element) + self._vins = self._vins.merge(means, on="Appellation", how="left") + + mean_col = f"mean_{element}" + self._vins[element] = self._vins[element].fillna(self._vins[mean_col]) - """ - tmp = df[["Appellation", col]].copy() + self._vins = self._vins.drop(columns=["mean_" + element]) + return self - tmp[col] = to_numeric(tmp[col], errors="coerce") - - # moyenne par appellation - means = tmp.groupby("Appellation", as_index=False)[col].mean() - - means[col] = means[col].fillna(0) - - means = means.rename(columns={col: f"mean_{col}"}) - - -def mean_robert(df: DataFrame) -> DataFrame: - return mean_score(df, "Robert") - - -def mean_robinson(df: DataFrame) -> DataFrame: - return mean_score(df, "Robinson") - - -def mean_suckling(df: DataFrame) -> DataFrame: - return mean_score(df, "Suckling") - - -def fill_missing_scores(df: DataFrame) -> DataFrame: - """ - Remplacer les notes manquantes par la moyenne - des vins de la même appellation. - """ - df_copy = df.copy() - df_copy["Appellation"] = df_copy["Appellation"].astype(str).str.strip() - - for score in SCORE_COLS: - df_copy[score] = to_numeric(df_copy[score], errors="coerce") - - temp_cols: list[str] = [] - - for score in SCORE_COLS: - mean_df = mean_score(df_copy, score) - mean_name = f"mean_{score}" - temp_cols.append(mean_name) - - df_copy = df_copy.merge(mean_df, on="Appellation", how="left") - df_copy[score] = df_copy[score].fillna(df_copy[mean_name]) - - df_copy = df_copy.drop(columns=temp_cols) - return df_copy - - -def encode_appellation(df: DataFrame, column: str = "Appellation") -> DataFrame: - """ - Remplace la colonne 'Appellation' par des colonnes indicatrices - """ - df_copy = df.copy() - - appellations = df_copy[column].astype(str).str.strip() - - appellation_dummies = get_dummies(appellations) - - df_copy = df_copy.drop(columns=[column]) - - return df_copy.join(appellation_dummies) + def encode_appellation(self, column: str = "Appellation") -> Cleaning: + """ + Remplace la colonne 'Appellation' par des colonnes indicatrices + """ + appellations = self._vins[column].astype(str).str.strip() + appellation_dummies = get_dummies(appellations) + self._vins = self._vins.drop(columns=[column]) + self._vins = self._vins.join(appellation_dummies) + return self \ No newline at end of file diff --git a/tests/test_cleaning.py b/tests/test_cleaning.py index d376ccf..166c52c 100755 --- a/tests/test_cleaning.py +++ b/tests/test_cleaning.py @@ -1,64 +1,68 @@ -import pandas as pd import pytest from pandas import DataFrame - -from cleaning import ( - SCORE_COLS, - drop_empty_appellation, - mean_score, - fill_missing_scores, - encode_appellation, -) +from unittest.mock import patch, mock_open +from cleaning import Cleaning @pytest.fixture -def df_raw() -> DataFrame: - return pd.DataFrame({ - "Appellation": ["Pauillac", "Pauillac ", "Margaux", None, "Pomerol", "Pomerol"], - "Robert": ["95", None, "bad", 90, None, None], - "Robinson": [None, "93", 18, None, None, None], - "Suckling": [96, None, None, None, 91, None], - "Prix": ["10.0", "11.0", "20.0", "30.0", "40.0", "50.0"], - }) +def cleaning_raw() -> Cleaning: + """ + "Appellation": ["Pauillac", "Pauillac ", "Margaux", None , "Pomerol", "Pomerol"], + "Robert": ["95" , None , "bad" , 90 , None , None ], + "Robinson": [None , "93" , 18 , None , None , None ], + "Suckling": [96 , None , None , None , 91 , None ], + "Prix": ["10.0" , "11.0" , "20.0" , "30.0", "40.0" , "50.0" ], + """ + csv_content = """Appellation,Robert,Robinson,Suckling,Prix +Pauillac,95,,96,10.0 +Pauillac ,,93,,11.0 +Margaux,bad,18,,20.0 +,90,,,30.0 +Pomerol,,,91,40.0 +Pomerol,,,,50.0 +""" + m = mock_open(read_data=csv_content) + with patch("builtins.open", m): + return Cleaning("donnee.csv") -def test_drop_empty_appellation(df_raw: DataFrame): - out = drop_empty_appellation(df_raw) +def test_drop_empty_appellation(cleaning_raw: Cleaning) -> None: + out = cleaning_raw.drop_empty_appellation().getVins() assert out["Appellation"].isna().sum() == 0 - assert len(out) == 5 + assert len(out) == 5 -def test_mean_score_zero_when_no_scores(df_raw: DataFrame): - out = drop_empty_appellation(df_raw) - m = mean_score(out, "Robert") +def test_mean_score_zero_when_no_scores(cleaning_raw: Cleaning) -> None: + out = cleaning_raw.drop_empty_appellation() + m = out._mean_score("Robert") assert list(m.columns) == ["Appellation", "mean_Robert"] - - # Pomerol n'a aucune note Robert => moyenne doit être 0 - pomerol_mean = m.loc[m["Appellation"].str.strip() == "Pomerol", "mean_Robert"].iloc[0] + pomerol_mean = m.loc[m["Appellation"].str.strip() == "Pomerol", "mean_Robert"].iloc[ + 0 + ] assert pomerol_mean == 0 -def test_fill_missing_scores(df_raw: DataFrame): - out = drop_empty_appellation(df_raw) - filled = fill_missing_scores(out) +def test_fill_missing_scores(cleaning_raw: Cleaning): + cleaning_raw._vins["Appellation"] = cleaning_raw._vins["Appellation"].str.strip() - # plus de NaN dans les colonnes de scores - for col in SCORE_COLS: + cleaning_raw.drop_empty_appellation() + filled = cleaning_raw.fill_missing_scores().getVins() + for col in cleaning_raw.SCORE_COLS: assert filled[col].isna().sum() == 0 - - assert filled.loc[1, "Robert"] == 95.0 - - # pas de colonnes temporaires mean_* - for col in SCORE_COLS: - assert f"mean_{col}" not in filled.columns + + pauillac_robert = filled[filled["Appellation"] == "Pauillac"]["Robert"] + assert (pauillac_robert == 95.0).all() -def test_encode_appellation(df_raw: DataFrame): - out = drop_empty_appellation(df_raw) - filled = fill_missing_scores(out) - encoded = encode_appellation(filled) +def test_encode_appellation(cleaning_raw: Cleaning): + cleaning_raw._vins["Appellation"] = cleaning_raw._vins["Appellation"].str.strip() - # la colonne texte disparaît - assert "Appellation" not in encoded.columns - assert "Pauillac" in encoded.columns - assert encoded.loc[0, "Pauillac"] == 1 \ No newline at end of file + out = ( + cleaning_raw.drop_empty_appellation() + .fill_missing_scores() + .encode_appellation() + .getVins() + ) + assert "Appellation" not in out.columns + assert "Pauillac" in out.columns + assert int(out.loc[0, "Pauillac"]) == 1 From f4ded6d8b56fd998f3d4af57c880344d2479347c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20GUEZO?= Date: Fri, 6 Mar 2026 21:02:52 +0100 Subject: [PATCH 11/12] ajout: correction d'erreur, changement de main dans cleaning --- src/cleaning.py | 36 ++++++++++++++++---- src/main.py | 58 --------------------------------- src/scraper.py | 74 +++++++++++++++++++++++++----------------- tests/test_cleaning.py | 3 +- 4 files changed, 75 insertions(+), 96 deletions(-) delete mode 100755 src/main.py diff --git a/src/cleaning.py b/src/cleaning.py index 781896b..4bf78bb 100755 --- a/src/cleaning.py +++ b/src/cleaning.py @@ -1,7 +1,14 @@ #!/usr/bin/env python3 -from typing import cast, override +from os import getcwd +from os.path import normpath, join +from typing import cast from pandas import DataFrame, read_csv, to_numeric, get_dummies +from sys import argv + + +def path_filename(filename: str) -> str: + return normpath(join(getcwd(), filename)) class Cleaning: @@ -18,7 +25,6 @@ class Cleaning: def getVins(self) -> DataFrame: return self._vins.copy(deep=True) - @override def __str__(self) -> str: """ Affiche un résumé du DataFrame @@ -34,7 +40,7 @@ class Cleaning: f"Statistiques numériques :\n{self._vins.describe().round(2)}\n\n" ) - def drop_empty_appellation(self) -> Cleaning: + def drop_empty_appellation(self) -> "Cleaning": self._vins = self._vins.dropna(subset=["Appellation"]) return self @@ -61,7 +67,7 @@ class Cleaning: def _mean_suckling(self) -> DataFrame: return self._mean_score("Suckling") - def fill_missing_scores(self) -> Cleaning: + def fill_missing_scores(self) -> "Cleaning": """ Remplacer les notes manquantes par la moyenne des vins de la même appellation. @@ -69,14 +75,14 @@ class Cleaning: for element in self.SCORE_COLS: means = self._mean_score(element) self._vins = self._vins.merge(means, on="Appellation", how="left") - + mean_col = f"mean_{element}" self._vins[element] = self._vins[element].fillna(self._vins[mean_col]) self._vins = self._vins.drop(columns=["mean_" + element]) return self - def encode_appellation(self, column: str = "Appellation") -> Cleaning: + def encode_appellation(self, column: str = "Appellation") -> "Cleaning": """ Remplace la colonne 'Appellation' par des colonnes indicatrices """ @@ -84,4 +90,20 @@ class Cleaning: appellation_dummies = get_dummies(appellations) self._vins = self._vins.drop(columns=[column]) self._vins = self._vins.join(appellation_dummies) - return self \ No newline at end of file + return self + + +def main() -> None: + if len(argv) != 2: + raise ValueError(f"Usage: {argv[0]} ") + + filename = argv[1] + cleaning: Cleaning = Cleaning(filename) + _ = cleaning.drop_empty_appellation().fill_missing_scores().encode_appellation() + + +if __name__ == "__main__": + try: + main() + except Exception as e: + print(f"ERREUR: {e}") diff --git a/src/main.py b/src/main.py deleted file mode 100755 index 512fe20..0000000 --- a/src/main.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 - -from os import getcwd -from os.path import normpath, join -from sys import argv -from pandas import read_csv, DataFrame - -from cleaning import * - - -def load_csv(filename: str) -> DataFrame: - path: str = normpath(join(getcwd(), filename)) - return read_csv(path) - - -def save_csv(df: DataFrame, out_filename: str) -> None: - df.to_csv(out_filename, index=False) - - -def main() -> None: - if len(argv) != 2: - raise ValueError(f"Usage: {argv[0]} ") - - df = load_csv(argv[1]) - - display_info(df, "Avant le nettoyage") - - df = drop_empty_appellation(df) - save_csv(df, "donnee_clean.csv") - display_info(df, "Après nettoyage d'appellations manquantes") - - #la moyenne des notes des vins pour chaque appellation - robert_means = mean_robert(df) - save_csv(robert_means, "mean_robert_by_appellation.csv") - display_info(robert_means, "Moyennes Robert par appellation") - - robinson_means = mean_robinson(df) - save_csv(robinson_means, "mean_robinson_by_appellation.csv") - display_info(robinson_means, "Moyennes Robinson par appellation") - - suckling_means = mean_suckling(df) - save_csv(suckling_means, "mean_suckling_by_appellation.csv") - display_info(suckling_means, "Moyennes Suckling par appellation") - - df_missing_scores = fill_missing_scores(df) - save_csv(df_missing_scores, "donnee_filled.csv") - display_info(df_missing_scores, "Après remplissage des notes manquantes par la moyenne de l'appellation") - - df_ready = encode_appellation(df_missing_scores) - save_csv(df_ready, "donnee_ready.csv") - display_info(df_ready, "Après remplacer la colonne 'Appellation' par des colonnes indicatrices") - - -if __name__ == "__main__": - try: - main() - except Exception as e: - print(f"ERREUR: {e}") \ No newline at end of file diff --git a/src/scraper.py b/src/scraper.py index bc599a6..76e4f6a 100755 --- a/src/scraper.py +++ b/src/scraper.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 from collections import OrderedDict -from io import SEEK_END, SEEK_SET, BufferedWriter +from io import SEEK_END, SEEK_SET, BufferedWriter, TextIOWrapper from json import JSONDecodeError, loads from os import makedirs from os.path import dirname, exists, join, normpath, realpath @@ -407,6 +407,44 @@ class Scraper: except (JSONDecodeError, HTTPError) as e: print(f"Erreur sur le produit {link}: {e}") + def _initstate(self, reset: bool) -> tuple[int, set[str]]: + """ + appelle la fonction pour load le cache, si il existe + pas, il utilise les variables de base sinon il override + toute les variables pour continuer et pas recommencer le + processus en entier. + + Args: + reset (bool): pouvoir le reset ou pas + + Returns: + tuple[int, set[str]]: le contenu de la page et du cache + """ + if not reset: + # + serializable: tuple[int, set[str]] | None = loadstate() + if isinstance(serializable, tuple): + return serializable + return 1, set() + + def _ensuretitle(self, f: TextIOWrapper, title: str) -> None: + """ + check si le titre est bien présent au début du buffer + sinon il l'ecrit, petit bug potentiel, a+ ecrit tout le + temps a la fin du buffer, si on a ecrit des choses avant + le titre sera apres ces données mais on part du principe + que personne va toucher le fichier. + + Args: + f (TextIOWrapper): buffer stream fichier + title (str): titre du csv + """ + _ = f.seek(0, SEEK_SET) + if not (f.read(len(title)) == title): + _ = f.write(title) + else: + _ = f.seek(0, SEEK_END) + def getvins(self, subdir: str, filename: str, reset: bool = False) -> None: """ Scrape toutes les pages d'une catégorie et sauvegarde en CSV. @@ -420,35 +458,13 @@ class Scraper: mode: Literal["w", "a+"] = "w" if reset else "a+" # titre title: str = "Appellation,Robert,Robinson,Suckling,Prix\n" - # page du début - page: int = 1 - # le set qui sert de cache - cache: set[str] = set[str]() + # page: page où commence le scraper + # cache: tout les pages déjà parcourir + page, cache = self._initstate(reset) - custom_format = "{l_bar} {bar:20} {r_bar}" - - if not reset: - # appelle la fonction pour load le cache, si il existe - # pas, il utilise les variables de base sinon il override - # toute les variables pour continuer et pas recommencer le - # processus en entier. - serializable: tuple[int, set[str]] | None = loadstate() - if isinstance(serializable, tuple): - # override la page et le cache - page, cache = serializable try: with open(filename, mode) as f: - # check si le titre est bien présent au début du buffer - # sinon il l'ecrit, petit bug potentiel, a+ ecrit tout le - # temps a la fin du buffer, si on a ecrit des choses avant - # le titre sera apres ces données mais on part du principe - # que personne va toucher le fichier. - _ = f.seek(0, SEEK_SET) - if not (f.read(len(title)) == title): - _ = f.write(title) - else: - _ = f.seek(0, SEEK_END) - + self._ensuretitle(f, title) while True: products_list: list[dict[str, Any]] | None = ( self._geturlproductslist(f"{subdir}?page={page}") @@ -457,7 +473,7 @@ class Scraper: break pbar: tqdm[dict[str, Any]] = tqdm( - products_list, bar_format=custom_format + products_list, bar_format="{l_bar} {bar:20} {r_bar}" ) for product in pbar: keyword: str = cast( @@ -469,7 +485,7 @@ class Scraper: self._writevins(cache, product, f) page += 1 # va créer un fichier au début et l'override - # tout les 5 pages au cas où SIGHUP ou autre + # tout les 5 pages au cas où SIGHUP ou autre if page % 5 == 0 and not reset: savestate((page, cache)) except (Exception, HTTPError, KeyboardInterrupt, JSONDecodeError): diff --git a/tests/test_cleaning.py b/tests/test_cleaning.py index 166c52c..28f42c6 100755 --- a/tests/test_cleaning.py +++ b/tests/test_cleaning.py @@ -1,5 +1,4 @@ import pytest -from pandas import DataFrame from unittest.mock import patch, mock_open from cleaning import Cleaning @@ -49,7 +48,7 @@ def test_fill_missing_scores(cleaning_raw: Cleaning): filled = cleaning_raw.fill_missing_scores().getVins() for col in cleaning_raw.SCORE_COLS: assert filled[col].isna().sum() == 0 - + pauillac_robert = filled[filled["Appellation"] == "Pauillac"]["Robert"] assert (pauillac_robert == 95.0).all() From de1d325fb72b79806ef4b76ab5309fe4f5c1a0fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20GUEZO?= Date: Fri, 6 Mar 2026 21:08:31 +0100 Subject: [PATCH 12/12] fix: enlever la generation de page --- .github/workflows/python-app.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 89ac80e..20b970f 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -36,10 +36,3 @@ jobs: - name: Test with pytest run: pytest - - - name: Deploy Doc - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - run: | - git config user.name github-actions - git config user.email github-actions@github.com - mkdocs gh-deploy --force