From dec9f414ac477e8b51e1cb68c879701c3285cf0c Mon Sep 17 00:00:00 2001 From: Emmy D'Anello Date: Sun, 9 Jun 2024 17:45:56 +0200 Subject: [PATCH] =?UTF-8?q?N=C3=A9cessit=C3=A9=20du=20mode=20debug=20pour?= =?UTF-8?q?=20un=20contenu=20verbeux?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 21 +++++++++------ nupes/scripts/export_resultats_2024.py | 31 ++++++++++++---------- nupes/scripts/import_candidats_2024.py | 18 ++++++------- nupes/scripts/import_geographie.py | 22 +++++++-------- nupes/scripts/import_resultats_web_2024.py | 30 ++++++++++++--------- 5 files changed, 68 insertions(+), 54 deletions(-) diff --git a/main.py b/main.py index 8306bc4..524b803 100755 --- a/main.py +++ b/main.py @@ -25,22 +25,27 @@ def main(): load_dotenv() args = parse_args() + debug = args.debug engine = create_engine(os.getenv("SQLALCHEMY_URL"), echo=args.debug) match args.action: case "import_geographie": - print(f"Import des données géographiques pour les élections {args.type} {args.year}") - import_geographie.run(engine) + if debug: + print(f"Import des données géographiques pour les élections {args.type} {args.year}") + import_geographie.run(engine, debug) case "import_candidats": - print(f"Import des candidats pour les élections {args.type} {args.year}") - import_candidats_2024.run(engine) + if debug: + print(f"Import des candidats pour les élections {args.type} {args.year}") + import_candidats_2024.run(engine, debug) case "import_resultats": - print(f"Import des résultats pour les élections {args.type} {args.year}") - import_resultats_web_2024.run(engine) + if debug: + print(f"Import des résultats pour les élections {args.type} {args.year}") + import_resultats_web_2024.run(engine, debug) case "export_resultats": - print(f"Export des résultats pour les élections {args.type} {args.year}") - export_resultats_2024.run(engine) + if debug: + print(f"Export des résultats pour les élections {args.type} {args.year}") + export_resultats_2024.run(engine, debug) case _: print(f"Action {args.action} non reconnue") diff --git a/nupes/scripts/export_resultats_2024.py b/nupes/scripts/export_resultats_2024.py index 67740c0..1e26b16 100644 --- a/nupes/scripts/export_resultats_2024.py +++ b/nupes/scripts/export_resultats_2024.py @@ -12,7 +12,7 @@ from nupes.models.europeennes2024 import Bloc, Nuance, Liste, \ DATA_DIR = Path(__file__).parent.parent / 'data' -def exporter_listes(engine: Engine) -> None: +def exporter_listes(engine: Engine, debug: bool = False) -> None: with Session(engine) as session: blocs = session.execute(select(Bloc)).scalars().all() blocs_json = [] @@ -64,7 +64,7 @@ def exporter_listes(engine: Engine) -> None: json.dump(listes_json, f) -def exporter_resultats_france(engine: Engine) -> None: +def exporter_resultats_france(engine: Engine, debug: bool = False) -> None: with Session(engine) as session: resultats_france = session.execute(select(ResultatsFrance)).scalar_one_or_none() if not resultats_france: @@ -96,12 +96,13 @@ def exporter_resultats_france(engine: Engine) -> None: session.commit() -def exporter_resultats_regions(engine: Engine) -> None: +def exporter_resultats_regions(engine: Engine, debug: bool = False) -> None: with Session(engine) as session: regions = session.execute(select(Region)).scalars().all() regions_json = [] - for region in tqdm(regions, desc="Régions"): + regions_iterator = tqdm(regions, desc="Régions") if debug else regions + for region in regions_iterator: region_json = {'code_insee': region.code_insee, 'nom': region.libelle, 'departements': [dpt.code_insee for dpt in region.departements]} regions_json.append(region_json) @@ -146,12 +147,13 @@ def exporter_resultats_regions(engine: Engine) -> None: json.dump(regions_json, f) -def exporter_resultats_departements(engine: Engine) -> None: +def exporter_resultats_departements(engine: Engine, debug: bool = False) -> None: with Session(engine) as session: departements = session.execute(select(Departement)).scalars().all() departements_json = [] - for departement in tqdm(departements, desc="Départements"): + iterator = tqdm(departements, desc="Départements") if debug else departements + for departement in iterator: departement_json = {'code_insee': departement.code_insee, 'nom': departement.libelle, 'region': departement.region_code, 'communes': [commune.code_insee for commune in departement.communes]} @@ -199,12 +201,13 @@ def exporter_resultats_departements(engine: Engine) -> None: json.dump(departements_json, f) -def exporter_resultats_communes(engine: Engine) -> None: +def exporter_resultats_communes(engine: Engine, debug: bool = False) -> None: with Session(engine) as session: communes = session.execute(select(Commune)).scalars().all() communes_json = [] - for commune in tqdm(communes, desc="Communes"): + iterator = tqdm(communes, desc="Communes") if debug else communes + for commune in iterator: commune_json = {'code_insee': commune.code_insee, 'nom': commune.libelle, 'departement': commune.departement_code, 'bureaux_vote': [bv.id for bv in commune.bureaux_vote]} @@ -252,10 +255,10 @@ def exporter_resultats_communes(engine: Engine) -> None: json.dump(communes_json, f) -def run(engine: Engine) -> None: - exporter_listes(engine) - exporter_resultats_france(engine) - exporter_resultats_regions(engine) - exporter_resultats_departements(engine) +def run(engine: Engine, debug: bool = False) -> None: + exporter_listes(engine, debug) + exporter_resultats_france(engine, debug) + exporter_resultats_regions(engine, debug) + exporter_resultats_departements(engine, debug) # FIXME Les communes prennent trop de temps - # exporter_resultats_communes(engine) + # exporter_resultats_communes(engine, debug) diff --git a/nupes/scripts/import_candidats_2024.py b/nupes/scripts/import_candidats_2024.py index e7a2c53..0a73531 100644 --- a/nupes/scripts/import_candidats_2024.py +++ b/nupes/scripts/import_candidats_2024.py @@ -8,7 +8,7 @@ from nupes.cache import get_file from nupes.models.europeennes2024 import Bloc, Nuance, Liste, Candidat -def creer_blocs(engine: Engine) -> None: +def creer_blocs(engine: Engine, debug: bool = False) -> None: blocs = [ {"id": 1, "nom": "Gauche", "couleur": "#BB1840"}, {"id": 2, "nom": "Droite libérale", "couleur": "#FFEB00"}, @@ -28,7 +28,7 @@ def creer_blocs(engine: Engine) -> None: session.commit() -def creer_nuances(engine: Engine) -> None: +def creer_nuances(engine: Engine, debug: bool = False) -> None: nuances = [ {"code": "LEXG", "nom": "Liste d'extrême gauche", "couleur": "#BB0000"}, {"code": "LCOM", "nom": "Liste du Parti communiste français", "couleur": "#DD0000"}, @@ -57,7 +57,7 @@ def creer_nuances(engine: Engine) -> None: session.commit() -def importer_listes(engine: Engine) -> None: +def importer_listes(engine: Engine, debug: bool = False) -> None: listes = [ {"numero": 1, "nom": "Pour une humanité souveraine", "nuance_id": "LDIV", "bloc_id": 5}, {"numero": 2, "nom": "Pour une démocratie réelle : Décidons nous-mêmes !", @@ -126,7 +126,7 @@ def importer_listes(engine: Engine) -> None: session.commit() -def importer_candidats(engine: Engine) -> None: +def importer_candidats(engine: Engine, debug: bool = False) -> None: DATASET_URL = "https://www.data.gouv.fr/fr/datasets/r/483cd4bd-1b0e-4b52-a923-aadadf9c8f62" file = get_file(DATASET_URL, "candidats_2024.csv") @@ -166,8 +166,8 @@ def importer_candidats(engine: Engine) -> None: session.commit() -def run(engine: Engine) -> None: - creer_blocs(engine) - creer_nuances(engine) - importer_listes(engine) - importer_candidats(engine) +def run(engine: Engine, debug: bool = False) -> None: + creer_blocs(engine, debug) + creer_nuances(engine, debug) + importer_listes(engine, debug) + importer_candidats(engine, debug) diff --git a/nupes/scripts/import_geographie.py b/nupes/scripts/import_geographie.py index 0818f89..c4edab4 100644 --- a/nupes/scripts/import_geographie.py +++ b/nupes/scripts/import_geographie.py @@ -9,7 +9,7 @@ from nupes.cache import get_file from nupes.models.geographie import BureauVote, Circonscription, Commune, Departement, Region -def importer_regions(engine: Engine) -> None: +def importer_regions(engine: Engine, debug: bool = False) -> None: etag = requests.get( "https://public.opendatasoft.com/api/explore/v2.1/catalog/datasets" "/georef-france-region?select=data_processed").json()['data_processed'] @@ -36,7 +36,7 @@ def importer_regions(engine: Engine) -> None: session.commit() -def importer_departements(engine: Engine) -> None: +def importer_departements(engine: Engine, debug: bool = False) -> None: etag = requests.get( "https://public.opendatasoft.com/api/explore/v2.1/catalog/datasets" "/georef-france-departement?select=data_processed").json()['data_processed'] @@ -66,7 +66,7 @@ def importer_departements(engine: Engine) -> None: session.commit() -def importer_communes(engine: Engine) -> None: +def importer_communes(engine: Engine, debug: bool = False) -> None: etag = requests.get( "https://public.opendatasoft.com/api/explore/v2.1/catalog/datasets" "/georef-france-commune?select=data_processed").json()['data_processed'] @@ -95,7 +95,7 @@ def importer_communes(engine: Engine) -> None: session.commit() -def importer_bureaux_vote(engine: Engine) -> None: +def importer_bureaux_vote(engine: Engine, debug: bool = False) -> None: etag = requests.get( "https://public.opendatasoft.com/api/explore/v2.1/catalog/datasets" "/elections-france-bureau-vote-2022?select=data_processed").json()['data_processed'] @@ -138,7 +138,7 @@ def importer_bureaux_vote(engine: Engine) -> None: session.commit() -def importer_contours_bureaux_vote(engine: Engine) -> None: +def importer_contours_bureaux_vote(engine: Engine, debug: bool = False) -> None: file = get_file("https://www.data.gouv.fr/fr/datasets/r/f98165a7-7c37-4705-a181-bcfc943edc73", "contours-bureaux-vote.geojson") @@ -169,9 +169,9 @@ def importer_contours_bureaux_vote(engine: Engine) -> None: session.commit() -def run(engine: Engine) -> None: - importer_regions(engine) - importer_departements(engine) - importer_communes(engine) - importer_bureaux_vote(engine) - importer_contours_bureaux_vote(engine) +def run(engine: Engine, debug: bool = False) -> None: + importer_regions(engine, debug) + importer_departements(engine, debug) + importer_communes(engine, debug) + importer_bureaux_vote(engine, debug) + importer_contours_bureaux_vote(engine, debug) diff --git a/nupes/scripts/import_resultats_web_2024.py b/nupes/scripts/import_resultats_web_2024.py index dc241d5..13b4326 100644 --- a/nupes/scripts/import_resultats_web_2024.py +++ b/nupes/scripts/import_resultats_web_2024.py @@ -13,7 +13,7 @@ from nupes.models.geographie import Region, Departement, Commune BASE_URL = "https://www.resultats-elections.interieur.gouv.fr/europeennes2024/ensemble_geographique" -def importer_resultats_france(engine: Engine) -> None: +def importer_resultats_france(engine: Engine, debug: bool = False) -> None: file = get_file(f"{BASE_URL}/index.html", "resultats2024/resultats.html") with file.open() as f: @@ -53,9 +53,11 @@ def importer_resultats_france(engine: Engine) -> None: session.commit() -def importer_resultats_regions(engine: Engine) -> None: +def importer_resultats_regions(engine: Engine, debug: bool = False) -> None: with Session(engine) as session: - for region in tqdm(session.execute(select(Region)).scalars().all(), desc="Régions"): + regions = session.execute(select(Region)).scalars().all() + regions_iterator = tqdm(regions, desc="Régions") if debug else regions + for region in regions_iterator: reg_code = region.code_insee if reg_code == "984" or reg_code == "989": # TAAF + Île de Clipperton, pas d'élection @@ -107,9 +109,11 @@ def importer_resultats_regions(engine: Engine) -> None: session.commit() -def importer_resultats_departements(engine: Engine) -> None: +def importer_resultats_departements(engine: Engine, debug: bool = False) -> None: with Session(engine) as session: - for dpt in tqdm(session.execute(select(Departement)).scalars().all(), desc="Départements"): + departements = session.execute(select(Departement)).scalars().all() + iterator = tqdm(departements, desc="Départements") if debug else departements + for dpt in iterator: reg_code = dpt.region_code reg_path = f"{reg_code}/" dpt_code = dpt.code_insee @@ -168,9 +172,11 @@ def importer_resultats_departements(engine: Engine) -> None: session.commit() -def importer_resultats_communes(engine: Engine) -> None: +def importer_resultats_communes(engine: Engine, debug: bool = False) -> None: with Session(engine) as session: - for commune in tqdm(session.execute(select(Commune)).scalars().all(), desc="Communes"): + communes = session.execute(select(Commune)).scalars().all() + iterator = tqdm(communes, desc="Communes") if debug else communes + for commune in iterator: reg_code = commune.departement.region_code reg_path = f"{reg_code}/" dpt_code = commune.departement_code @@ -290,9 +296,9 @@ def analyser_resultats(file) -> dict: return parsed_data -def run(engine: Engine) -> None: - importer_resultats_france(engine) - importer_resultats_regions(engine) - importer_resultats_departements(engine) +def run(engine: Engine, debug: bool = False) -> None: + importer_resultats_france(engine, debug) + importer_resultats_regions(engine, debug) + importer_resultats_departements(engine, debug) # FIXME Les communes prennent trop de temps - # importer_resultats_communes(engine) + # importer_resultats_communes(engine, debug)