#!/usr/bin/python3
# -*- coding: utf-8 -*-
########################################################################
#
# This file is part of python module <pyspc>.
# Copyright (C) 2013-2021 R. Marty
# (renaud.marty@developpement-durable.gouv.fr)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program (see COPYING.txt).
# If not, see <http://www.gnu.org/licenses/>.
#
########################################################################
"""
Données d'observation et de prévision - Prévisions du SPC LCI - Version 2017
"""
import collections
from datetime import datetime as dt, timedelta as td
import pandas as pnd
import pyspc.core.exception as _exception
from pyspc.io.dbase.mdb import Mdb
from pyspc.convention.prevision import (
P17_DATATYPES,
P17_SQL_META, P17_SQL_DATA,
P17_TABLES_META, P17_ATTS_META,
P17_TABLES_DATA, P17_ATTS_DATA
)
[docs]
class Prevision17(Mdb):
"""
Classe destinée à traiter la base Prévision du SPC LCI, période 2017-2019
Attributes
----------
filename : str
Chemin de la base de données
sql : str
Requête courante au format SQL
"""
[docs]
def __init__(self, filename=None):
"""
Initialisation de l'instance Prevision17
Parameters
----------
filename : str
Chemin de la base de données
"""
super().__init__(filename=filename)
[docs]
def read_fcst(self, codes=None, valid=False, released=False,
hydro_version='hydro3', first_dt=None, last_dt=None,
warning=True):
"""
Récupération des prévisions de la base Prevision17
Parameters
----------
codes : list
Liste des identifiants des stations
valid : bool
Prévision expertisée (True). Défaut: False
released : bool
Prévision diffusée (True). Défaut: False
hydro_version : str
Référentiel : 'hydro2' ou 'hydro3'
first_dt : datetime
Premier pas de temps des données
last_dt : datetime
Dernier pas de temps des données
warning : bool
Afficher les avertissements ? défaut: True
Returns
-------
pnd.DataFrame
Tableau des données
See Also
--------
pyspc.data.prevision.Prevision17.get_fcst_hydro3
"""
# ---------------------------------------------------------------------
# 0- Contrôles
# ---------------------------------------------------------------------
_exception.check_listlike(codes)
_exception.check_bool(valid)
_exception.check_bool(released)
# ---------------------------------------------------------------------
# 1- Version Hydro3
# ---------------------------------------------------------------------
if hydro_version == 'hydro3':
return self.read_fcst_hydro3(
codes=codes, valid=valid, released=released,
first_dt=first_dt, last_dt=last_dt, warning=warning)
# ---------------------------------------------------------------------
# 9- Erreur
# ---------------------------------------------------------------------
raise ValueError('Version incorrecte du référentiel Hydro: '
'Hydro-3 seulement')
[docs]
def read_fcst_hydro3(self, codes=None, valid=False, released=False,
first_dt=None, last_dt=None, warning=True):
"""
Récupération des prévisions de la base Prevision17.
Parameters
----------
codes : list
Liste des identifiants des stations
valid : bool
Prévision expertisée (True). Défaut: False
released : bool
Prévision diffusée (True). Défaut: False
first_dt : datetime
Premier pas de temps des données
last_dt : datetime
Dernier pas de temps des données
warning : bool
Afficher les avertissements ? défaut: True
Returns
-------
pnd.DataFrame
Tableau des données
Examples
--------
>>> from datetime import datetime as dt
>>> from pyspc.data.prevision import Prevision17
>>> f = 'data/io/dbase/prevision_2017.mdb'
>>> reader = Prevision17(filename=f)
Exemple de prévision validée
>>> content = reader.read_fcst(
... codes=['K0403010'],
... first_dt=dt(2016, 11, 21, 12),
... last_dt=dt(2016, 11, 21, 12),
... valid=True,
... released=True
... )
>>> content
K0403010 ...
2016-11-21 12:00:00 ...
2003 ...
pilote ...
1 ...
Val ... Val90conv
DateVal ...
2016-11-21 14:00:00 5.337000 ... 0.489486
2016-11-21 15:00:00 5.512000 ... 0.511436
2016-11-21 16:00:00 5.458000 ... 0.526863
2016-11-21 17:00:00 5.307000 ... 0.538143
2016-11-21 18:00:00 5.103000 ... 0.546724
2016-11-21 19:00:00 5.214000 ... 0.571114
2016-11-21 20:00:00 5.599000 ... 0.619568
2016-11-21 21:00:00 6.171000 ... 0.689225
2016-11-21 22:00:00 7.422000 ... 0.797537
2016-11-21 23:00:00 9.638000 ... 0.960593
2016-11-22 00:00:00 12.877000 ... 1.179920
2016-11-22 01:00:00 16.025000 ... 1.381224
2016-11-22 02:00:00 18.378000 ... 1.533681
2016-11-22 03:00:00 19.707001 ... 1.615447
2016-11-22 04:00:00 20.676001 ... 1.685737
2016-11-22 05:00:00 21.527000 ... 1.753344
2016-11-22 06:00:00 22.257999 ... 1.818271
2016-11-22 07:00:00 24.580000 ... 1.928249
2016-11-22 08:00:00 29.033001 ... 2.119796
2016-11-22 09:00:00 35.544998 ... 2.378982
2016-11-22 10:00:00 42.487999 ... 2.640653
2016-11-22 11:00:00 49.056000 ... 2.880322
2016-11-22 12:00:00 55.080002 ... 3.096919
2016-11-22 13:00:00 58.733002 ... 3.219622
2016-11-22 14:00:00 58.667999 ... 3.229056
2016-11-22 15:00:00 55.799000 ... 3.152230
2016-11-22 16:00:00 51.264999 ... 3.019641
2016-11-22 17:00:00 45.583000 ... 2.842086
2016-11-22 18:00:00 39.361000 ... 2.634085
2016-11-22 19:00:00 34.172001 ... 2.442746
... ... ... ...
2016-11-23 08:00:00 9.410000 ... 1.242212
2016-11-23 09:00:00 8.764000 ... 1.193289
2016-11-23 10:00:00 8.184000 ... 1.148385
2016-11-23 11:00:00 7.661000 ... 1.107038
2016-11-23 12:00:00 7.190000 ... 1.068896
2016-11-23 13:00:00 6.765000 ... 1.033426
2016-11-23 14:00:00 6.379000 ... 1.000886
2016-11-23 15:00:00 6.027000 ... 0.970794
2016-11-23 16:00:00 5.704000 ... 0.942849
2016-11-23 17:00:00 5.407000 ... 0.916812
2016-11-23 18:00:00 5.134000 ... 0.892577
2016-11-23 19:00:00 4.887000 ... 0.869784
2016-11-23 20:00:00 4.658000 ... 0.848397
2016-11-23 21:00:00 4.446000 ... 0.828371
2016-11-23 22:00:00 4.250000 ... 0.809792
2016-11-23 23:00:00 4.068000 ... 0.792524
2016-11-24 00:00:00 3.897000 ... 0.776336
2016-11-24 01:00:00 3.744000 ... 0.762966
2016-11-24 02:00:00 3.609000 ... 0.751447
2016-11-24 03:00:00 3.483000 ... 0.740582
2016-11-24 04:00:00 3.365000 ... 0.730292
2016-11-24 05:00:00 3.253000 ... 0.720438
2016-11-24 06:00:00 3.148000 ... 0.711030
2016-11-24 07:00:00 3.056000 ... 0.702234
2016-11-24 08:00:00 2.970000 ... 0.694036
2016-11-24 09:00:00 2.891000 ... 0.686579
2016-11-24 10:00:00 2.816000 ... 0.679495
2016-11-24 11:00:00 2.745000 ... 0.672769
2016-11-24 12:00:00 2.677000 ... 0.666299
2016-11-24 13:00:00 2.616000 ... 0.659047
Exemple de prévision brute
>>> content = reader.read_fcst(
... codes=['K0403010'],
... first_dt=dt(2016, 11, 21, 12),
... last_dt=dt(2016, 11, 21, 12),
... valid=False,
... released=False
... )
>>> content
K0403010 ...
2016-11-21 12:00:00 ...
2001 ... 2009
Val ... Val90conv
DateVal ...
2016-11-21 14:00:00 5.301000 ... 0.490555
2016-11-21 15:00:00 5.391000 ... 0.515114
2016-11-21 16:00:00 5.240000 ... 0.534334
2016-11-21 17:00:00 5.018000 ... 0.549011
2016-11-21 18:00:00 4.779000 ... 0.560499
2016-11-21 19:00:00 4.753000 ... 0.591624
2016-11-21 20:00:00 4.898000 ... 0.652424
2016-11-21 21:00:00 5.152000 ... 0.740339
2016-11-21 22:00:00 5.676000 ... 0.878463
2016-11-21 23:00:00 6.571000 ... 1.090107
2016-11-22 00:00:00 7.992000 ... 1.375605
2016-11-22 01:00:00 9.519000 ... 1.608265
2016-11-22 02:00:00 10.753000 ... 1.739104
2016-11-22 03:00:00 11.653000 ... 1.801832
2016-11-22 04:00:00 12.417000 ... 1.845703
2016-11-22 05:00:00 12.939000 ... 1.888197
2016-11-22 06:00:00 13.574000 ... 1.929369
2016-11-22 07:00:00 15.139000 ... 2.011298
2016-11-22 08:00:00 17.966999 ... 2.168612
2016-11-22 09:00:00 22.056000 ... 2.387600
2016-11-22 10:00:00 26.490999 ... 2.608593
2016-11-22 11:00:00 30.813000 ... 2.810109
2016-11-22 12:00:00 34.910999 ... 2.991875
2016-11-22 13:00:00 37.716999 ... 3.091040
2016-11-22 14:00:00 38.320000 ... 3.090423
2016-11-22 15:00:00 37.151001 ... 3.013842
2016-11-22 16:00:00 34.826000 ... 2.888248
2016-11-22 17:00:00 31.632000 ... 2.723027
2016-11-22 18:00:00 27.943001 ... 2.530869
2016-11-22 19:00:00 24.782000 ... 2.353826
... ... ... ...
2016-11-23 08:00:00 7.914000 ... 1.217304
2016-11-23 09:00:00 7.414000 ... 1.170411
2016-11-23 10:00:00 6.961000 ... 1.127340
2016-11-23 11:00:00 6.550000 ... 1.087598
2016-11-23 12:00:00 6.178000 ... 1.050925
2016-11-23 13:00:00 5.839000 ... 1.016824
2016-11-23 14:00:00 5.528000 ... 0.985592
2016-11-23 15:00:00 5.242000 ... 0.956564
2016-11-23 16:00:00 4.979000 ... 0.929556
2016-11-23 17:00:00 4.736000 ... 0.904402
2016-11-23 18:00:00 4.510000 ... 0.881127
2016-11-23 19:00:00 4.306000 ... 0.859018
2016-11-23 20:00:00 4.116000 ... 0.838292
2016-11-23 21:00:00 3.940000 ... 0.818897
2016-11-23 22:00:00 3.781000 ... 0.801056
2016-11-23 23:00:00 3.644000 ... 0.784292
2016-11-24 00:00:00 3.516000 ... 0.768602
2016-11-24 01:00:00 3.394000 ... 0.756737
2016-11-24 02:00:00 3.280000 ... 0.745581
2016-11-24 03:00:00 3.171000 ... 0.735027
2016-11-24 04:00:00 3.069000 ... 0.724997
2016-11-24 05:00:00 2.973000 ... 0.715378
2016-11-24 06:00:00 2.883000 ... 0.706276
2016-11-24 07:00:00 2.806000 ... 0.697734
2016-11-24 08:00:00 2.733000 ... 0.689927
2016-11-24 09:00:00 2.664000 ... 0.682656
2016-11-24 10:00:00 2.602000 ... 0.675762
2016-11-24 11:00:00 2.543000 ... 0.669222
2016-11-24 12:00:00 2.488000 ... 0.662596
2016-11-24 13:00:00 2.435000 ... 0.655113
"""
# ---------------------------------------------------------------------
# 0- Contrôles
# ---------------------------------------------------------------------
_exception.check_listlike(codes)
_exception.check_bool(valid)
_exception.check_bool(released)
# ---------------------------------------------------------------------
# 1- Appliquer la requête SQL SERIE
# ---------------------------------------------------------------------
series = self.read_serie_hydro3(
codes=codes, first_dt=first_dt, last_dt=last_dt,
valid=valid, released=released, warning=warning)
if self.check_sql_return(content=series, warning=warning) is None:
return None
# ---------------------------------------------------------------------
# 2- Appliquer la requête SQL DATA
# ---------------------------------------------------------------------
# ---------------------------------------------------------------------
# 2.1- Connexion base Access
# ---------------------------------------------------------------------
self.connect()
# ---------------------------------------------------------------------
# 2.2- Rechercher les prévisions
# ---------------------------------------------------------------------
dfs = {}
for meta, nserie in series.items():
self._set_sql_fcst_hydro3(nserie=nserie, valid=valid)
content = self.execute(warning=warning)
if self.check_sql_return(content=content, warning=warning) is None:
continue
# ---------------------------------------------------------------------
# 2.3- Tableau de données
# ---------------------------------------------------------------------
# Créer le pnd.DataFrame
df = {c: [x[k+1] for x in content]
for k, c in enumerate(P17_ATTS_DATA[valid])}
df = pnd.DataFrame(df)
# Convertir les échéances en dates
c = P17_ATTS_DATA[valid][0]
df[c] = df[c].apply(lambda x: x*td(hours=1)) + meta[1]
# Définir l'index par la colonnes DateVal
df = df.set_index(keys=c, drop=True)
dfs[meta] = df
# ---------------------------------------------------------------------
# 2.4- Déconnexion base Access
# ---------------------------------------------------------------------
self.close()
# ---------------------------------------------------------------------
# 3- Création du DataFrame global
# ---------------------------------------------------------------------
df = pnd.concat(dfs, axis=1).sort_index(axis=1)
# ---------------------------------------------------------------------
# 4- Retour
# ---------------------------------------------------------------------
return df
[docs]
def read_serie_hydro3(self, codes=None, first_dt=None, last_dt=None,
valid=False, released=False, warning=True):
"""
Récupération des identifiants des séries de la base Prevision17.
Parameters
----------
codes : list
Liste des identifiants des stations
valid : bool
Prévision expertisée (True). Défaut: False
released : bool
Prévision diffusée (True). Défaut: False
first_dt : datetime
Premier pas de temps des données
last_dt : datetime
Dernier pas de temps des données
warning : bool
Afficher les avertissements ? défaut: True
Returns
-------
dict
Dictionnaire des identifiants des séries
- clé : ("CodeStation", "CodeModele", "DateDerObs") si brute
- clé : ("CodeStation", "CodeModele", "DateDerObs",
"Source", "Diffuse") si expertisée
- valeur : identifiant de la série
None
si aucune série ne correspond aux paramètres
"""
# ---------------------------------------------------------------------
# 0- Contrôles
# ---------------------------------------------------------------------
_exception.check_listlike(codes)
_exception.check_bool(valid)
_exception.check_bool(released)
# ---------------------------------------------------------------------
# 1- Connexion base Access
# ---------------------------------------------------------------------
self.connect()
# ---------------------------------------------------------------------
# 2- Appliquer la requête SQL SERIE
# ---------------------------------------------------------------------
self._set_sql_serie_hydro3(
codes=codes, first_dt=first_dt, last_dt=last_dt,
valid=valid, released=released)
series = self.execute(warning=warning)
# ---------------------------------------------------------------------
# 3- Déconnexion base Access
# ---------------------------------------------------------------------
self.close()
# ---------------------------------------------------------------------
# 4- Traitement du résultat de la requête SQL
# ---------------------------------------------------------------------
if self.check_sql_return(content=series, warning=warning) is None:
return None
data = collections.OrderedDict()
for serie in series:
meta = tuple(serie[1:])
data[meta] = serie[0]
# ---------------------------------------------------------------------
# 5- Retour
# ---------------------------------------------------------------------
return data
def _set_sql_serie_hydro3(self, codes=None, first_dt=None,
last_dt=None, valid=False, released=False):
"""
Définir de la requête SQL destinée à récupérer les séries
Parameters
----------
codes : list
Liste des identifiants des stations
valid : bool
Prévision expertisée (True). Défaut: False
released : bool
Prévision diffusée (True). Défaut: False
first_dt : datetime
Premier pas de temps des données
last_dt : datetime
Dernier pas de temps des données
"""
# ---------------------------------------------------------------------
# 0- Contrôles
# ---------------------------------------------------------------------
_exception.check_listlike(codes)
_exception.check_bool(valid)
_exception.check_bool(released)
if not valid and released:
released = False
# ---------------------------------------------------------------------
# 1- Requête SQL
# ---------------------------------------------------------------------
stations = ",".join(["'" + code + "'" for code in codes])
self.sql = P17_SQL_META[(valid, released)].format(
P17_TABLES_META[valid],
*P17_ATTS_META[valid],
stations
)
# ---------------------------------------------------------------------
# 2- Complément date
# ---------------------------------------------------------------------
if isinstance(first_dt, dt):
x = self.from_datetime(first_dt, "%Y%m%d%H", -60)
self.sql += f" AND ({P17_TABLES_META[valid]}.{P17_ATTS_META[valid][2]} >= {x})"
if isinstance(last_dt, dt):
x = self.from_datetime(last_dt, "%Y%m%d%H", +60)
self.sql += f" AND ({P17_TABLES_META[valid]}.{P17_ATTS_META[valid][2]} <= {x})"
def _set_sql_fcst_hydro3(self, nserie=None, valid=False):
"""
Définir de la requête SQL pour récupérer les chroniques de prévision.
Parameters
----------
nserie : int
Identifiant de la série
valid : bool
Prévision expertisée (True). Défaut: False
"""
# ---------------------------------------------------------------------
# 0- Contrôles
# ---------------------------------------------------------------------
_exception.check_int(nserie)
_exception.check_bool(valid)
# ---------------------------------------------------------------------
# 1- Requête SQL
# ---------------------------------------------------------------------
self.sql = P17_SQL_DATA.format(
P17_TABLES_DATA[valid],
*P17_ATTS_DATA[valid],
nserie
)
[docs]
@classmethod
def get_datatypes(cls):
"""
Type de données des bases Prevision 2017
Returns
-------
list
Type de données des bases Prevision 2017
"""
return sorted(P17_DATATYPES)