ynerant/nupes-elections
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Nécessité du mode debug pour un contenu verbeux

Browse Source
This commit is contained in:
Emmy D'Anello 2024-06-09 17:45:56 +02:00
parent aabd3f0153
commit dec9f414ac
Signed by: ynerant
GPG Key ID: 3A75C55819C8CF85
5 changed files with 68 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
main.py
View File

@ -25,22 +25,27 @@ def main():
load_dotenv()
args = parse_args()
debug = args.debug
engine = create_engine(os.getenv("SQLALCHEMY_URL"), echo=args.debug)
match args.action:
case "import_geographie":
print(f"Import des données géographiques pour les élections {args.type} {args.year}")
import_geographie.run(engine)
if debug:
print(f"Import des données géographiques pour les élections {args.type} {args.year}")
import_geographie.run(engine, debug)
case "import_candidats":
print(f"Import des candidats pour les élections {args.type} {args.year}")
import_candidats_2024.run(engine)
if debug:
print(f"Import des candidats pour les élections {args.type} {args.year}")
import_candidats_2024.run(engine, debug)
case "import_resultats":
print(f"Import des résultats pour les élections {args.type} {args.year}")
import_resultats_web_2024.run(engine)
if debug:
print(f"Import des résultats pour les élections {args.type} {args.year}")
import_resultats_web_2024.run(engine, debug)
case "export_resultats":
print(f"Export des résultats pour les élections {args.type} {args.year}")
export_resultats_2024.run(engine)
if debug:
print(f"Export des résultats pour les élections {args.type} {args.year}")
export_resultats_2024.run(engine, debug)
case _:
print(f"Action {args.action} non reconnue")

31
nupes/scripts/export_resultats_2024.py
View File

@ -12,7 +12,7 @@ from nupes.models.europeennes2024 import Bloc, Nuance, Liste, \
DATA_DIR = Path(__file__).parent.parent / 'data'
def exporter_listes(engine: Engine) -> None:
def exporter_listes(engine: Engine, debug: bool = False) -> None:
with Session(engine) as session:
blocs = session.execute(select(Bloc)).scalars().all()
blocs_json = []
@ -64,7 +64,7 @@ def exporter_listes(engine: Engine) -> None:
json.dump(listes_json, f)
def exporter_resultats_france(engine: Engine) -> None:
def exporter_resultats_france(engine: Engine, debug: bool = False) -> None:
with Session(engine) as session:
resultats_france = session.execute(select(ResultatsFrance)).scalar_one_or_none()
if not resultats_france:
@ -96,12 +96,13 @@ def exporter_resultats_france(engine: Engine) -> None:
session.commit()
def exporter_resultats_regions(engine: Engine) -> None:
def exporter_resultats_regions(engine: Engine, debug: bool = False) -> None:
with Session(engine) as session:
regions = session.execute(select(Region)).scalars().all()
regions_json = []
for region in tqdm(regions, desc="Régions"):
regions_iterator = tqdm(regions, desc="Régions") if debug else regions
for region in regions_iterator:
region_json = {'code_insee': region.code_insee, 'nom': region.libelle,
'departements': [dpt.code_insee for dpt in region.departements]}
regions_json.append(region_json)
@ -146,12 +147,13 @@ def exporter_resultats_regions(engine: Engine) -> None:
json.dump(regions_json, f)
def exporter_resultats_departements(engine: Engine) -> None:
def exporter_resultats_departements(engine: Engine, debug: bool = False) -> None:
with Session(engine) as session:
departements = session.execute(select(Departement)).scalars().all()
departements_json = []
for departement in tqdm(departements, desc="Départements"):
iterator = tqdm(departements, desc="Départements") if debug else departements
for departement in iterator:
departement_json = {'code_insee': departement.code_insee, 'nom': departement.libelle,
'region': departement.region_code,
'communes': [commune.code_insee for commune in departement.communes]}
@ -199,12 +201,13 @@ def exporter_resultats_departements(engine: Engine) -> None:
json.dump(departements_json, f)
def exporter_resultats_communes(engine: Engine) -> None:
def exporter_resultats_communes(engine: Engine, debug: bool = False) -> None:
with Session(engine) as session:
communes = session.execute(select(Commune)).scalars().all()
communes_json = []
for commune in tqdm(communes, desc="Communes"):
iterator = tqdm(communes, desc="Communes") if debug else communes
for commune in iterator:
commune_json = {'code_insee': commune.code_insee, 'nom': commune.libelle,
'departement': commune.departement_code,
'bureaux_vote': [bv.id for bv in commune.bureaux_vote]}
@ -252,10 +255,10 @@ def exporter_resultats_communes(engine: Engine) -> None:
json.dump(communes_json, f)
def run(engine: Engine) -> None:
exporter_listes(engine)
exporter_resultats_france(engine)
exporter_resultats_regions(engine)
exporter_resultats_departements(engine)
def run(engine: Engine, debug: bool = False) -> None:
exporter_listes(engine, debug)
exporter_resultats_france(engine, debug)
exporter_resultats_regions(engine, debug)
exporter_resultats_departements(engine, debug)
# FIXME Les communes prennent trop de temps
# exporter_resultats_communes(engine)
# exporter_resultats_communes(engine, debug)

18
nupes/scripts/import_candidats_2024.py
View File

@ -8,7 +8,7 @@ from nupes.cache import get_file
from nupes.models.europeennes2024 import Bloc, Nuance, Liste, Candidat
def creer_blocs(engine: Engine) -> None:
def creer_blocs(engine: Engine, debug: bool = False) -> None:
blocs = [
{"id": 1, "nom": "Gauche", "couleur": "#BB1840"},
{"id": 2, "nom": "Droite libérale", "couleur": "#FFEB00"},
@ -28,7 +28,7 @@ def creer_blocs(engine: Engine) -> None:
session.commit()
def creer_nuances(engine: Engine) -> None:
def creer_nuances(engine: Engine, debug: bool = False) -> None:
nuances = [
{"code": "LEXG", "nom": "Liste d'extrême gauche", "couleur": "#BB0000"},
{"code": "LCOM", "nom": "Liste du Parti communiste français", "couleur": "#DD0000"},
@ -57,7 +57,7 @@ def creer_nuances(engine: Engine) -> None:
session.commit()
def importer_listes(engine: Engine) -> None:
def importer_listes(engine: Engine, debug: bool = False) -> None:
listes = [
{"numero": 1, "nom": "Pour une humanité souveraine", "nuance_id": "LDIV", "bloc_id": 5},
{"numero": 2, "nom": "Pour une démocratie réelle : Décidons nous-mêmes !",
@ -126,7 +126,7 @@ def importer_listes(engine: Engine) -> None:
session.commit()
def importer_candidats(engine: Engine) -> None:
def importer_candidats(engine: Engine, debug: bool = False) -> None:
DATASET_URL = "https://www.data.gouv.fr/fr/datasets/r/483cd4bd-1b0e-4b52-a923-aadadf9c8f62"
file = get_file(DATASET_URL, "candidats_2024.csv")
@ -166,8 +166,8 @@ def importer_candidats(engine: Engine) -> None:
session.commit()
def run(engine: Engine) -> None:
creer_blocs(engine)
creer_nuances(engine)
importer_listes(engine)
importer_candidats(engine)
def run(engine: Engine, debug: bool = False) -> None:
creer_blocs(engine, debug)
creer_nuances(engine, debug)
importer_listes(engine, debug)
importer_candidats(engine, debug)

22
nupes/scripts/import_geographie.py
View File

@ -9,7 +9,7 @@ from nupes.cache import get_file
from nupes.models.geographie import BureauVote, Circonscription, Commune, Departement, Region
def importer_regions(engine: Engine) -> None:
def importer_regions(engine: Engine, debug: bool = False) -> None:
etag = requests.get(
"https://public.opendatasoft.com/api/explore/v2.1/catalog/datasets"
"/georef-france-region?select=data_processed").json()['data_processed']
@ -36,7 +36,7 @@ def importer_regions(engine: Engine) -> None:
session.commit()
def importer_departements(engine: Engine) -> None:
def importer_departements(engine: Engine, debug: bool = False) -> None:
etag = requests.get(
"https://public.opendatasoft.com/api/explore/v2.1/catalog/datasets"
"/georef-france-departement?select=data_processed").json()['data_processed']
@ -66,7 +66,7 @@ def importer_departements(engine: Engine) -> None:
session.commit()
def importer_communes(engine: Engine) -> None:
def importer_communes(engine: Engine, debug: bool = False) -> None:
etag = requests.get(
"https://public.opendatasoft.com/api/explore/v2.1/catalog/datasets"
"/georef-france-commune?select=data_processed").json()['data_processed']
@ -95,7 +95,7 @@ def importer_communes(engine: Engine) -> None:
session.commit()
def importer_bureaux_vote(engine: Engine) -> None:
def importer_bureaux_vote(engine: Engine, debug: bool = False) -> None:
etag = requests.get(
"https://public.opendatasoft.com/api/explore/v2.1/catalog/datasets"
"/elections-france-bureau-vote-2022?select=data_processed").json()['data_processed']
@ -138,7 +138,7 @@ def importer_bureaux_vote(engine: Engine) -> None:
session.commit()
def importer_contours_bureaux_vote(engine: Engine) -> None:
def importer_contours_bureaux_vote(engine: Engine, debug: bool = False) -> None:
file = get_file("https://www.data.gouv.fr/fr/datasets/r/f98165a7-7c37-4705-a181-bcfc943edc73",
"contours-bureaux-vote.geojson")
@ -169,9 +169,9 @@ def importer_contours_bureaux_vote(engine: Engine) -> None:
session.commit()
def run(engine: Engine) -> None:
importer_regions(engine)
importer_departements(engine)
importer_communes(engine)
importer_bureaux_vote(engine)
importer_contours_bureaux_vote(engine)
def run(engine: Engine, debug: bool = False) -> None:
importer_regions(engine, debug)
importer_departements(engine, debug)
importer_communes(engine, debug)
importer_bureaux_vote(engine, debug)
importer_contours_bureaux_vote(engine, debug)

30
nupes/scripts/import_resultats_web_2024.py
View File

@ -13,7 +13,7 @@ from nupes.models.geographie import Region, Departement, Commune
BASE_URL = "https://www.resultats-elections.interieur.gouv.fr/europeennes2024/ensemble_geographique"
def importer_resultats_france(engine: Engine) -> None:
def importer_resultats_france(engine: Engine, debug: bool = False) -> None:
file = get_file(f"{BASE_URL}/index.html", "resultats2024/resultats.html")
with file.open() as f:
@ -53,9 +53,11 @@ def importer_resultats_france(engine: Engine) -> None:
session.commit()
def importer_resultats_regions(engine: Engine) -> None:
def importer_resultats_regions(engine: Engine, debug: bool = False) -> None:
with Session(engine) as session:
for region in tqdm(session.execute(select(Region)).scalars().all(), desc="Régions"):
regions = session.execute(select(Region)).scalars().all()
regions_iterator = tqdm(regions, desc="Régions") if debug else regions
for region in regions_iterator:
reg_code = region.code_insee
if reg_code == "984" or reg_code == "989":
# TAAF + Île de Clipperton, pas d'élection
@ -107,9 +109,11 @@ def importer_resultats_regions(engine: Engine) -> None:
session.commit()
def importer_resultats_departements(engine: Engine) -> None:
def importer_resultats_departements(engine: Engine, debug: bool = False) -> None:
with Session(engine) as session:
for dpt in tqdm(session.execute(select(Departement)).scalars().all(), desc="Départements"):
departements = session.execute(select(Departement)).scalars().all()
iterator = tqdm(departements, desc="Départements") if debug else departements
for dpt in iterator:
reg_code = dpt.region_code
reg_path = f"{reg_code}/"
dpt_code = dpt.code_insee
@ -168,9 +172,11 @@ def importer_resultats_departements(engine: Engine) -> None:
session.commit()
def importer_resultats_communes(engine: Engine) -> None:
def importer_resultats_communes(engine: Engine, debug: bool = False) -> None:
with Session(engine) as session:
for commune in tqdm(session.execute(select(Commune)).scalars().all(), desc="Communes"):
communes = session.execute(select(Commune)).scalars().all()
iterator = tqdm(communes, desc="Communes") if debug else communes
for commune in iterator:
reg_code = commune.departement.region_code
reg_path = f"{reg_code}/"
dpt_code = commune.departement_code
@ -290,9 +296,9 @@ def analyser_resultats(file) -> dict:
return parsed_data
def run(engine: Engine) -> None:
importer_resultats_france(engine)
importer_resultats_regions(engine)
importer_resultats_departements(engine)
def run(engine: Engine, debug: bool = False) -> None:
importer_resultats_france(engine, debug)
importer_resultats_regions(engine, debug)
importer_resultats_departements(engine, debug)
# FIXME Les communes prennent trop de temps
# importer_resultats_communes(engine)
# importer_resultats_communes(engine, debug)