Upgrade FFVL provider with the new API. Waiting for encoding fix on their side...

Author: ysavary

.env.localhost.template

@@ -1,27 +1,15 @@
-# COMMONS
+# Commons
 MONGODB_URL=mongodb://localhost:8011/winds
 REDIS_URL=redis://localhost:8012/0
-ADMIN_DB_URL=postgres://postgres:postgres@localhost:8016/winds
-
 GOOGLE_API_KEY=
 
-# PROVIDERS
-
-# Windline
-WINDLINE_SQL_URL=
-
-# METAR
-CHECKWX_API_KEY=
-
-# Romma
-ROMMA_KEY=
-
-# iWeathar
-IWEATHAR_KEY=
-
-# BornToFly
+# Providers
+ADMIN_DB_URL=postgres://postgres:postgres@localhost:8016/winds
 BORN_TO_FLY_VENDOR_ID=
 BORN_TO_FLY_DEVICE_ID=
-
-# Windy
+CHECKWX_API_KEY=
+FFVL_API_KEY=
+IWEATHAR_KEY=
+ROMMA_KEY=
+WINDLINE_SQL_URL=
 WINDY_API_KEY=

.env.template

@@ -1,27 +1,15 @@
-# COMMONS
+# Commons
 MONGODB_URL=mongodb://mongodb:27017/winds
 REDIS_URL=redis://redis:6379/0
-ADMIN_DB_URL=postgres://postgres:postgres@postgres:5432/winds
-
 GOOGLE_API_KEY=
 
-# PROVIDERS
-
-# Windline
-WINDLINE_SQL_URL=
-
-# METAR
-CHECKWX_API_KEY=
-
-# Romma
-ROMMA_KEY=
-
-# iWeathar
-IWEATHAR_KEY=
-
-# BornToFly
+# Providers
+ADMIN_DB_URL=postgres://postgres:postgres@postgres:5432/winds
 BORN_TO_FLY_VENDOR_ID=
 BORN_TO_FLY_DEVICE_ID=
-
-# Windy
+CHECKWX_API_KEY=
+FFVL_API_KEY=
+IWEATHAR_KEY=
+ROMMA_KEY=
+WINDLINE_SQL_URL=
 WINDY_API_KEY=

providers/ffvl.py

@@ -1,27 +1,34 @@
-import logging
+import json
 
 import arrow
 import requests
 from dateutil import tz
-from tenacity import after_log, retry, stop_after_delay, wait_random_exponential
 
+from settings import FFVL_API_KEY
 from winds_mobi_provider import Pressure, Provider, ProviderException, StationStatus
 
 
 class Ffvl(Provider):
     provider_code = "ffvl"
     provider_name = "ffvl.fr"
-    provider_url = "http://www.balisemeteo.com"
+    provider_url = "https://www.balisemeteo.com"
+
+    def __init__(self, ffvl_api_key):
+        super().__init__()
+        self.ffvl_api_key = ffvl_api_key
 
     def process_data(self):
         stations = {}
         try:
             self.log.info("Processing FFVL data...")
 
             result = requests.get(
-                "http://data.ffvl.fr/json/balises.json", timeout=(self.connect_timeout, self.read_timeout)
+                f"https://data.ffvl.fr/api/?base=balises&r=list&mode=json&key={self.ffvl_api_key}",
+                timeout=(self.connect_timeout, self.read_timeout),
             )
-            ffvl_stations = result.json()
+            # TODO: remove the BOM encoding when the FFVL will fix the forbidden json encoding on their side
+            # https://www.rfc-editor.org/rfc/rfc7159#section-8.1
+            ffvl_stations = json.loads(result.content.decode("utf-8-sig"))
 
             for ffvl_station in ffvl_stations:
                 ffvl_id = None
@@ -52,20 +59,13 @@ def process_data(self):
             self.log.exception(f"Error while processing stations: {e}")
 
         try:
-
-            @retry(
-                wait=wait_random_exponential(multiplier=2, min=2),
-                stop=stop_after_delay(60),
-                after=after_log(self.log, logging.WARNING),
+            result = requests.get(
+                f"https://data.ffvl.fr/api/?base=balises&r=releves_meteo&key={self.ffvl_api_key}",
+                timeout=(self.connect_timeout, self.read_timeout),
             )
-            def request_data():
-                # data.ffvl.fr randomly returns an empty file instead the json doc
-                result = requests.get(
-                    "http://data.ffvl.fr/json/relevesmeteo.json", timeout=(self.connect_timeout, self.read_timeout)
-                )
-                return result.json()
-
-            ffvl_measures = request_data()
+            # TODO: remove the BOM encoding when the FFVL will fix the forbidden json encoding on their side
+            # https://www.rfc-editor.org/rfc/rfc7159#section-8.1
+            ffvl_measures = json.loads(result.content.decode("utf-8-sig"))
 
             ffvl_tz = tz.gettz("Europe/Paris")
             for ffvl_measure in ffvl_measures:
@@ -107,7 +107,7 @@ def request_data():
 
 
 def ffvl():
-    Ffvl().process_data()
+    Ffvl(FFVL_API_KEY).process_data()
 
 
 if __name__ == "__main__":

settings.py

@@ -1,6 +1,6 @@
 import os
 
-# COMMONS
+# Commons
 MONGODB_URL = os.environ.get("MONGODB_URL") or "mongodb://localhost:27017/winds_mobi"
 REDIS_URL = os.environ.get("REDIS_URL") or "redis://localhost:6379/0"
 GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
@@ -9,24 +9,13 @@
 SENTRY_URL = os.environ.get("SENTRY_URL")
 ENVIRONMENT = os.environ.get("ENVIRONMENT") or "development"
 
-# PROVIDERS
+# Providers
 ADMIN_DB_URL = os.environ.get("ADMIN_DB_URL") or "postgres://postgres:postgres@localhost:5432/winds_mobi"
-
-# Windline
-WINDLINE_SQL_URL = os.environ.get("WINDLINE_SQL_URL")
-
-# METAR
-CHECKWX_API_KEY = os.environ.get("CHECKWX_API_KEY")
-
-# Romma
-ROMMA_KEY = os.environ.get("ROMMA_KEY")
-
-# iWeathar
-IWEATHAR_KEY = os.environ.get("IWEATHAR_KEY")
-
-# BornToFly
 BORN_TO_FLY_VENDOR_ID = os.environ.get("BORN_TO_FLY_VENDOR_ID")
 BORN_TO_FLY_DEVICE_ID = os.environ.get("BORN_TO_FLY_DEVICE_ID")
-
-# Windy
+CHECKWX_API_KEY = os.environ.get("CHECKWX_API_KEY")  # For METAR provider
+FFVL_API_KEY = os.environ.get("FFVL_API_KEY")
+IWEATHAR_KEY = os.environ.get("IWEATHAR_KEY")
+ROMMA_KEY = os.environ.get("ROMMA_KEY")
+WINDLINE_SQL_URL = os.environ.get("WINDLINE_SQL_URL")
 WINDY_API_KEY = os.environ.get("WINDY_API_KEY")