guezoloic/millesima-ai-engine
1
0
Fork 0
mirror of https://github.com/guezoloic/millesima-ai-engine.git synced 2026-03-28 18:03:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: sauvegarde toute les 5 pages

Browse Source
This commit is contained in:
Loïc GUEZO
2026-03-05 20:00:23 +01:00
parent 5303d36988
commit 8047b06253
1 changed files with 8 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/scraper.py
View File

@@ -384,8 +384,7 @@ class Scraper:
list[dict[str, Any]], data.get("products") list[dict[str, Any]], data.get("products")
) )
if isinstance(products, list): return products
return products
except (JSONDecodeError, HTTPError): except (JSONDecodeError, HTTPError):
return None return None
@@ -461,12 +460,18 @@ class Scraper:
products_list, bar_format=custom_format products_list, bar_format=custom_format
) )
for product in pbar: for product in pbar:
keyword = product.get("seoKeyword", "Inconnu")[:40] keyword: str = cast(
str, product.get("seoKeyword", "Inconnu")[:40]
)
pbar.set_description( pbar.set_description(
f"Page: {page:<3} | Product: {keyword:<40}" f"Page: {page:<3} | Product: {keyword:<40}"
) )
self._writevins(cache, product, f) self._writevins(cache, product, f)
page += 1 page += 1
# va créer un fichier au début et l'override
# tout les 5 pages au cas où SIGHUP ou autre
if page % 5 == 0 and not reset:
savestate((page, cache))
except (Exception, HTTPError, KeyboardInterrupt, JSONDecodeError): except (Exception, HTTPError, KeyboardInterrupt, JSONDecodeError):
if not reset: if not reset:
savestate((page, cache)) savestate((page, cache))