mirror of
https://github.com/guezoloic/millesima-ai-engine.git
synced 2026-03-30 10:46:26 +00:00
Compare commits
29 Commits
jalon2-loi
...
dependabot
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f29130f4bb | ||
|
|
615418f347 | ||
| 83c53dd6b0 | |||
| fc3f516cdd | |||
| 235275cfab | |||
| faca333cbf | |||
| f4dd93e4b0 | |||
| 7a4e49684f | |||
| f223acdfe6 | |||
| 7cd24bf6cb | |||
|
|
a75769eb3b | ||
| de513fca15 | |||
|
|
f87ea357f4 | ||
| 68dffa6486 | |||
| c7d2077b23 | |||
| 106877a073 | |||
|
|
416cfcbf8b | ||
| 32c5310e37 | |||
| 9dfc7457a0 | |||
| f5d5703e49 | |||
| 888defb6b6 | |||
| 734e3898e9 | |||
| 4bb3112dd0 | |||
| 54e4b7860b | |||
| b865a59aba | |||
|
|
fde1f36148 | ||
| 6fbb36ea37 | |||
|
|
bcacd7a915 | ||
|
|
d182e08f9b |
18
.github/dependabot.yml
vendored
Normal file
18
.github/dependabot.yml
vendored
Normal file
@@ -0,0 +1,18 @@
|
||||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "pip"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "saturday"
|
||||
open-pull-requests-limit: 5
|
||||
groups:
|
||||
python-dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
|
||||
6
.github/workflows/python-app.yml
vendored
6
.github/workflows/python-app.yml
vendored
@@ -19,15 +19,15 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python 3.10
|
||||
- name: Set up Python 3.x
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.10"
|
||||
python-version: "3.x"
|
||||
|
||||
- name: install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install ".[test,doc]"
|
||||
pip install ".[test]"
|
||||
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
|
||||
57
.github/workflows/static.yml
vendored
Normal file
57
.github/workflows/static.yml
vendored
Normal file
@@ -0,0 +1,57 @@
|
||||
# Simple workflow for deploying static content to GitHub Pages
|
||||
name: Deploy static content to Pages
|
||||
|
||||
on:
|
||||
# Runs on pushes targeting the default branch
|
||||
push:
|
||||
branches: ["main"]
|
||||
|
||||
# Allows you to run this workflow manually from the Actions tab
|
||||
workflow_dispatch:
|
||||
|
||||
# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
|
||||
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
|
||||
concurrency:
|
||||
group: "pages"
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
# Single deploy job since we're just deploying
|
||||
deploy:
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python 3.x
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.x'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -e ".[doc]"
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v5
|
||||
|
||||
- name: Build Documentation
|
||||
run: mkdocs build
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
# Upload entire repository
|
||||
path: './site'
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
21
LICENSE
Normal file
21
LICENSE
Normal file
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Loïc GUEZO and chahrazad DAHMANI
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
38
README.md
38
README.md
@@ -1 +1,37 @@
|
||||
# millesima_projetS6
|
||||
# Millesima AI Engine 🍷
|
||||
|
||||
> A **University of Paris-Est Créteil (UPEC)** Semester 6 project.
|
||||
|
||||
## Documentation
|
||||
- 🇫🇷 [Version Française](https://millesima-ai.github.guezoloic.com)
|
||||
> note: only french version enabled for now.
|
||||
---
|
||||
|
||||
## Installation
|
||||
> Make sure you have **Python 3.10+** installed.
|
||||
|
||||
1. **Clone the repository:**
|
||||
```bash
|
||||
git clone https://github.com/guezoloic/millesima-ai-engine.git
|
||||
cd millesima-ai-engine
|
||||
```
|
||||
|
||||
2. **Set up a virtual environment:**
|
||||
```bash
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
||||
```
|
||||
|
||||
3. **Install dependencies:**
|
||||
```bash
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### 1. Data Extraction (Scraping)
|
||||
To fetch the latest wine data from Millesima:
|
||||
```bash
|
||||
python3 src/scraper.py
|
||||
```
|
||||
> Note: that will take some time to fetch all data depending on the catalog size.
|
||||
17
docs/cleaning.md
Normal file
17
docs/cleaning.md
Normal file
@@ -0,0 +1,17 @@
|
||||
# Cleaning
|
||||
|
||||
## Sommaire
|
||||
[TOC]
|
||||
|
||||
---
|
||||
|
||||
## Classe `Cleaning`
|
||||
::: src.cleaning.Cleaning
|
||||
options:
|
||||
heading_level: 3
|
||||
members:
|
||||
- __init__
|
||||
- getVins
|
||||
- drop_empty_appellation
|
||||
- fill_missing_scores
|
||||
- encode_appellation
|
||||
@@ -1 +1,16 @@
|
||||
# Millesima
|
||||
|
||||
L’objectif de ce projet est d’étudier, en utilisant des méthodes d’apprentissage automatique, l’impact de différents critères (notes des critiques, appelation) sur le prix d’un vin. Pour ce faire, on s’appuiera sur le site Millesima (https://www.millesima.fr/), qui a l’avantage de ne pas posséder de protection contre les bots. Par respect pour l’hébergeur du site, on veillera à limiter au maximum le nombre de requêtes. En particulier, on s’assurera d’avoir un code fonctionnel avant de scraper l’intégralité du site, pour éviter les répétitions.
|
||||
|
||||
## projet
|
||||
<div style="text-align: center;">
|
||||
<object
|
||||
data="/projet.pdf"
|
||||
type="application/pdf"
|
||||
width="100%"
|
||||
height="1000px"
|
||||
>
|
||||
<p>Votre navigateur ne peut pas afficher ce PDF.
|
||||
<a href="/projet.pdf">Cliquez ici pour le télécharger.</a></p>
|
||||
</object>
|
||||
</div>
|
||||
1287
docs/learning.ipynb
Normal file
1287
docs/learning.ipynb
Normal file
File diff suppressed because one or more lines are too long
Binary file not shown.
@@ -1,3 +1,31 @@
|
||||
# Scraper
|
||||
|
||||
## Sommaire
|
||||
[TOC]
|
||||
|
||||
---
|
||||
|
||||
## Classe `Scraper`
|
||||
::: scraper.Scraper
|
||||
options:
|
||||
members:
|
||||
- __init__
|
||||
- getvins
|
||||
- getjsondata
|
||||
- getresponse
|
||||
- getsoup
|
||||
heading_level: 4
|
||||
|
||||
## Classe `_ScraperData`
|
||||
::: scraper._ScraperData
|
||||
options:
|
||||
members:
|
||||
- __init__
|
||||
- getdata
|
||||
- appellation
|
||||
- parker
|
||||
- robinson
|
||||
- suckling
|
||||
- prix
|
||||
- informations
|
||||
heading_level: 4
|
||||
@@ -1,4 +0,0 @@
|
||||
|
||||
# _ScraperData
|
||||
|
||||
::: scraper._ScraperData
|
||||
@@ -1,4 +1,5 @@
|
||||
site_name: "Projet Millesima S6"
|
||||
site_url: "https://millesima-ai.github.guezoloic.com"
|
||||
|
||||
theme:
|
||||
name: "material"
|
||||
@@ -6,6 +7,12 @@ theme:
|
||||
plugins:
|
||||
- search
|
||||
- mkdocstrings
|
||||
- mkdocs-jupyter
|
||||
|
||||
extra:
|
||||
generator: false
|
||||
|
||||
copyright: "Loïc GUEZO & Chahrazad DAHMANI – UPEC S6 – 2026"
|
||||
|
||||
markdown_extensions:
|
||||
- admonition
|
||||
|
||||
@@ -1,16 +1,28 @@
|
||||
[project]
|
||||
name = "projet-millesima-s6"
|
||||
name = "millesima-project-s6"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"requests==2.32.5",
|
||||
"requests==2.33.0",
|
||||
"beautifulsoup4==4.14.3",
|
||||
"pandas==2.3.3",
|
||||
"pandas==3.0.1",
|
||||
"tqdm==4.67.3",
|
||||
"scikit-learn==1.8.0",
|
||||
"matplotlib==3.10.8",
|
||||
"seaborn==0.13.2"
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
pythonpath = "src"
|
||||
testpaths = ["tests"]
|
||||
|
||||
[project.optional-dependencies]
|
||||
test = ["pytest==8.4.2", "requests-mock==1.12.1", "flake8==7.3.0"]
|
||||
doc = ["mkdocs<2.0.0", "mkdocs-material==9.6.23", "mkdocstrings[python]"]
|
||||
test = ["pytest==9.0.2", "requests-mock==1.12.1", "flake8==7.3.0"]
|
||||
doc = [
|
||||
"mkdocs<2.0.0",
|
||||
"mkdocs-material==9.7.6",
|
||||
"mkdocstrings[python]",
|
||||
"mkdocs-jupyter==0.26.1",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools", "wheel"]
|
||||
|
||||
@@ -92,14 +92,25 @@ class Cleaning:
|
||||
self._vins = self._vins.join(appellation_dummies)
|
||||
return self
|
||||
|
||||
def drop_empty_price(self) -> "Cleaning":
|
||||
self._vins = self._vins.dropna(subset=["Prix"])
|
||||
return self
|
||||
|
||||
def main() -> None:
|
||||
if len(argv) != 2:
|
||||
raise ValueError(f"Usage: {argv[0]} <filename.csv>")
|
||||
|
||||
filename = argv[1]
|
||||
cleaning: Cleaning = Cleaning(filename)
|
||||
_ = cleaning.drop_empty_appellation().fill_missing_scores().encode_appellation()
|
||||
def main(filename: str | None = None) -> None:
|
||||
if not filename:
|
||||
if len(argv) != 2:
|
||||
raise ValueError(f"Usage: {argv[0]} <filename.csv>")
|
||||
filename = argv[1]
|
||||
|
||||
cleaning: Cleaning = (
|
||||
Cleaning(filename)
|
||||
.drop_empty_appellation()
|
||||
.fill_missing_scores()
|
||||
.encode_appellation()
|
||||
.drop_empty_price()
|
||||
)
|
||||
cleaning.getVins().to_csv("clean.csv", index=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
64
src/learning.py
Executable file
64
src/learning.py
Executable file
@@ -0,0 +1,64 @@
|
||||
# from typing import Any, Callable
|
||||
# from pandas import DataFrame
|
||||
# from sklearn.linear_model import LinearRegression
|
||||
# from sklearn.preprocessing import StandardScaler
|
||||
# from sklearn.model_selection import train_test_split
|
||||
# from sklearn.pipeline import make_pipeline
|
||||
# import matplotlib.pyplot as plt
|
||||
|
||||
# from cleaning import Cleaning
|
||||
|
||||
|
||||
# class Learning:
|
||||
# def __init__(self, vins: DataFrame, target: str) -> None:
|
||||
# self.X = vins.drop(target, axis=1)
|
||||
# self.y = vins[target]
|
||||
|
||||
# self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(
|
||||
# self.X, self.y, test_size=0.25, random_state=49
|
||||
# )
|
||||
|
||||
# def evaluate(
|
||||
# self,
|
||||
# estimator,
|
||||
# pretreatment=None,
|
||||
# fn_score=lambda m, xt, yt: m.score(xt, yt),
|
||||
# ):
|
||||
|
||||
# pipeline = make_pipeline(pretreatment, estimator) if pretreatment else estimator
|
||||
# pipeline.fit(self.X_train, self.y_train)
|
||||
# score = fn_score(pipeline, self.X_test, self.y_test)
|
||||
# prediction = pipeline.predict(self.X_test)
|
||||
|
||||
# return score, prediction
|
||||
|
||||
# def draw(self, predictions, y_actual):
|
||||
# plt.figure(figsize=(8, 6))
|
||||
|
||||
# plt.scatter(
|
||||
# predictions,
|
||||
# y_actual,
|
||||
# alpha=0.5,
|
||||
# c="royalblue",
|
||||
# edgecolors="k",
|
||||
# label="Vins",
|
||||
# )
|
||||
|
||||
# mn = min(predictions.min(), y_actual.min())
|
||||
# mx = max(predictions.max(), y_actual.max())
|
||||
# plt.plot(
|
||||
# [mn, mx],
|
||||
# [mn, mx],
|
||||
# color="red",
|
||||
# linestyle="--",
|
||||
# lw=2,
|
||||
# label="Prédiction Parfaite",
|
||||
# )
|
||||
|
||||
# plt.xlabel("Prix estimés (estim_LR)")
|
||||
# plt.ylabel("Prix réels (y_test)")
|
||||
# plt.title("titre")
|
||||
# plt.legend()
|
||||
# plt.grid(True, linestyle=":", alpha=0.6)
|
||||
|
||||
# plt.show()
|
||||
@@ -377,9 +377,6 @@ class Scraper:
|
||||
try:
|
||||
data: dict[str, object] = self.getjsondata(subdir).getdata()
|
||||
|
||||
for element in ["initialReduxState", "categ", "content"]:
|
||||
data = cast(dict[str, object], data.get(element))
|
||||
|
||||
products: list[dict[str, Any]] = cast(
|
||||
list[dict[str, Any]], data.get("products")
|
||||
)
|
||||
@@ -493,11 +490,12 @@ class Scraper:
|
||||
savestate((page, cache))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(argv) != 3:
|
||||
raise ValueError(f"{argv[0]} <filename> <sous-url>")
|
||||
filename = argv[1]
|
||||
suburl = argv[2]
|
||||
def main(filename: str | None = None, suburl: str | None = None) -> None:
|
||||
if filename is None or suburl is None:
|
||||
if len(argv) != 3:
|
||||
raise ValueError(f"Usage: python {argv[0]} <filename> <sous-url>")
|
||||
filename = argv[1]
|
||||
suburl = argv[2]
|
||||
|
||||
scraper: Scraper = Scraper()
|
||||
scraper.getvins(suburl, filename)
|
||||
|
||||
@@ -185,17 +185,11 @@ def mock_site():
|
||||
{dumps({
|
||||
"props": {
|
||||
"pageProps": {
|
||||
"initialReduxState": {
|
||||
"categ": {
|
||||
"content": {
|
||||
"products": [
|
||||
{"seoKeyword": "/nino-negri-5-stelle-sfursat-2022.html",},
|
||||
{"seoKeyword": "/poubelle",},
|
||||
{"seoKeyword": "/",}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"products": [
|
||||
{"seoKeyword": "/nino-negri-5-stelle-sfursat-2022.html",},
|
||||
{"seoKeyword": "/poubelle",},
|
||||
{"seoKeyword": "/",}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -213,14 +207,8 @@ def mock_site():
|
||||
{dumps({
|
||||
"props": {
|
||||
"pageProps": {
|
||||
"initialReduxState": {
|
||||
"categ": {
|
||||
"content": {
|
||||
"products": [
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"products": [
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user