feat: initialise tests with dev db seed

Author: isMattCoding

.gitignore

@@ -136,3 +136,6 @@ GitHub.sublime-settings
 !.vscode/launch.json
 !.vscode/extensions.json
 .history
+
+test_seed_data.csv
+db.sqlite3

bikes/tests/__init__.py

@@ -1,2 +1,2 @@
-from model import *
-from views import *
+from .model import *
+from .views import *

bikes/tests/model.py

@@ -12,7 +12,6 @@ def setUp(self):
 
     def test_bike_creation(self):
         # Fetch the bike object to ensure it was created correctly
-        bike = Bikes.objects.get(date_time="2023-04-30 14:30:00+00")
-        print(bike.date_time)
+        bike = Bikes.objects.get(date_time="2023-04-30 14:00:00+00")
         self.assertEqual(bike.forecast, Decimal('103.12345'))
-        self.assertEqual(bike.date_time.strftime("%Y-%m-%d %H:%M:%S"), "2023-04-30 14:30:00")
+        self.assertEqual(bike.date_time.strftime("%Y-%m-%d %H:%M:%S"), "2023-04-30 14:00:00")

bikes/tests/views.py

@@ -1,22 +1,71 @@
 from django.test import TestCase
 from django.utils import timezone
 from bikes.views import Bikes
-from datetime import datetime
+from bikes.utils.views import *
+from test_helpers import *
+from datetime import datetime, time, timedelta
 from decimal import Decimal
+import csv
+import os.path
 
 class BikesViewsTestCase(TestCase):
-    # def setUp(self):
-    #     dt = timezone.make_aware(datetime(2023, 4, 30, 14, 00, 0))
-    #     Bikes.objects.create(actual=100.0, forecast=103.12345, upper=123.12345, lower=78.12345, date_time=dt)
+
+    def setUp(self):
+        reader = csv.DictReader(open('test_seed_data.csv'))
+        day_num = 0
+        day = timezone.now() - timedelta(days=day_num+1)
+        for row in reader:
+            if day_num == row['day']:
+                dt = timezone.make_aware(datetime.combine(day.date(), time(row['hour'], 0, 0)))
+                Bikes.objects.create(actual=row['actual'], forecast=row['forecast'], upper=row['upper'], lower=row['lower'], date_time=dt)
+            else:
+                day_num = int(row['day'])
+                day = timezone.now() - timedelta(days=day_num+1)
+                dt = timezone.make_aware(datetime.combine(day.date(), time(int(row['hour']), 0, 0)))
+                if row['actual'] == '':
+                    row['actual'] = 0
+                Bikes.objects.create(
+                    actual=row['actual'],
+                    forecast=Decimal(row['forecast']), upper=Decimal(row['upper']), lower=Decimal(row['lower']), date_time=dt)
+
+
 
 
-    # def test_bike_creation(self):
-    #     # Fetch the bike object to ensure it was created correctly
-    #     bike = Bikes.objects.get(date_time="2023-04-30 14:30:00+00")
-    #     print(bike.date_time)
-    #     self.assertEqual(bike.forecast, Decimal('103.12345'))
-    #     self.assertEqual(bike.date_time.strftime("%Y-%m-%d %H:%M:%S"), "2023-04-30 14:30:00")
 
     # def test_get_data_from_open_data(self):
-    #     b = Bikes()
-    print('toto')
+    #     """Let's see if we can get data from open data"""
+    #     data = get_data_from_open_data(id_compteur='100057445-103057445')
+    #     """We should be receiving data"""
+    #     self.assertNotEqual(data, "Waiting...")
+    #     """The data should be a list"""
+    #     self.assertIsInstance(data, list)
+    #     """Each entry in the list should have a 'sum_counts' and a 'date'"""
+    #     for obj in data:
+    #         self.assertTrue("sum_counts" in obj, f"'sum_counts' attribute is missing for {obj}")
+    #         self.assertTrue("date" in obj, f"'date' attribute is missing for {obj}")
+
+    async def test_get_treated_data(self):
+        treated_data = await get_treated_data()
+        print('hello',treated_data)
+        self.assertNotEqual(treated_data, "Waiting...")
+
+    # async def test_e2e(self):
+    #     """Let's see if we can save the results to the DB"""
+    #     context = {}
+    #     data = get_data_from_open_data(id_compteur='100057445-103057445')
+    #     treated_data = await get_treated_data()
+    #     for item in data:
+    #         item['DateTime'] = change_date_string_to_datetime(item['date'])
+    #     today = pd.to_datetime(data[10]['DateTime']).tz_convert('Europe/Paris')
+    #     is_entry_today = await check_is_entry_today(today)
+    #     if not is_entry_today:
+    #         MLResult = runMLModel(merge_data(data, treated_data))[-72:]
+    #         await saveResultsToDB(MLResult)
+    #         for item in MLResult:
+    #             item['DateTime'] = item['DateTime'].strftime("%Y-%m-%d %H:%M:%S")
+    #         context["forecast"] = MLResult
+    #     else:
+    #         sorted_treated_data = sorted(treated_data, key=lambda x: x['DateTime'])
+    #         for item in sorted_treated_data:
+    #             item['DateTime'] = item['DateTime'].strftime("%Y-%m-%d %H:%M:%S")
+    #         context["forecast"] = sorted_treated_data[-72:]

bikes/utils/views.py

@@ -1,5 +1,13 @@
+import math
+import pandas as pd
 import requests, time
 import datetime
+from tzlocal import get_localzone
+from asgiref.sync import sync_to_async
+from pytz import NonExistentTimeError
+from bikes.models import Bikes
+from greykite.framework.templates.autogen.forecast_config import ForecastConfig, MetadataParam, ModelComponentsParam
+from greykite.framework.templates.forecaster import Forecaster
 
 def get_data_from_open_data(date=None, id_compteur='100057445-103057445'):
     url = "https://opendata.paris.fr/api/explore/v2.1/catalog/datasets/comptage-velo-donnees-compteurs/records"
@@ -26,3 +34,115 @@ def get_data_from_open_data(date=None, id_compteur='100057445-103057445'):
         return data
     else:
         return "Waiting..."
+
+@sync_to_async
+def saveResultsToDB(result):
+    local_tz = get_localzone()  # Get local timezone
+    for data in result:
+        actual_data = None
+        if not math.isnan(data['actual']):
+            actual_data = data["actual"]
+        try:
+            # Convert string to datetime object and localize to 'Europe/Paris' timezone
+            date_time = pd.to_datetime(data["DateTime"]).tz_localize('Europe/Paris')
+        except NonExistentTimeError:
+            # Handle NonExistentTimeError by using local timezone instead
+            date_time = pd.to_datetime(data["DateTime"]).tz_localize(local_tz, ambiguous='NaT')
+        result_to_save = Bikes(
+            actual=actual_data,
+            forecast=data["forecast"],
+            upper=data["forecast_upper"],
+            lower=data["forecast_lower"],
+            date_time=date_time
+        )
+        print(result_to_save.__dict__)
+        result_to_save.save_result()
+
+def merge_data(data, treated_data):
+    filtered_data = list(filter(lambda entry: entry['actual'] is not None, treated_data))
+    treated_dates = {item['DateTime'] for item in filtered_data}
+    unique_data = [item for item in data if item['DateTime'] not in treated_dates]
+    abridged_treated_data = []
+    for item in filtered_data:
+        abridged_treated_data.append({'DateTime': item['DateTime'], 'sum_counts': item['actual']})
+    merged_data = abridged_treated_data + unique_data
+    return sorted(merged_data, key=lambda x: x['DateTime'])
+
+def runMLModel(data):
+    forecast_config = ForecastConfig(
+        model_template = "SILVERKITE",
+        forecast_horizon = 24,
+        coverage = 0.85,
+        metadata_param = MetadataParam(
+            time_col = "DateTimeCol",
+            value_col = "sum_counts",
+            freq = "H",
+            train_end_date=datetime.datetime.today()
+        )
+    )
+    df = pd.DataFrame(data)
+    df['DateTime'] = pd.to_datetime(df['DateTime'], utc=True, errors='coerce')
+    df['DateTime'] = df['DateTime'].values
+    df.sort_values(by='DateTime', inplace=True)
+    df['DateTimeCol'] = df['DateTime']
+    df.set_index('DateTime', inplace=True)
+
+    forecaster = Forecaster()
+    silverkite = forecaster.run_forecast_config(
+        df = df,
+        config = forecast_config
+    )
+
+    forecast_data_json  = silverkite.forecast.df.to_dict()
+
+    last_24_entries = {}
+    for forecast_key in forecast_data_json:
+        if forecast_key == "DateTime":
+            for y in forecast_data_json[forecast_key]:
+                forecast_data_json[forecast_key][y] = forecast_data_json[forecast_key][y].strftime("%Y-%m-%d %X")
+            forecast_value = dict(list(forecast_data_json[forecast_key].items())[-100:])
+        else :
+            forecast_value = dict(list(forecast_data_json[forecast_key].items())[-100:])
+        for index_key in forecast_value:
+            if index_key in last_24_entries:
+                last_24_entries[index_key][forecast_key] = forecast_value[index_key]
+            else:
+                last_24_entries[index_key] = {forecast_key: forecast_value[index_key]}
+
+    result = list(last_24_entries.values())
+    for item in result:
+        if 'DateTimeCol' in item:
+            item['DateTime'] = item.pop('DateTimeCol')
+
+    return result
+
+@sync_to_async
+def get_treated_data():
+    bikes_queryset = Bikes.objects.all()
+    data = []
+    for bikes_data in bikes_queryset:
+        actual_data = None
+        if bikes_data.actual != None:
+            actual_data = float(bikes_data.actual)
+        data.append({
+            "actual":actual_data,
+            "forecast":float(bikes_data.forecast),
+            "forecast_upper":float(bikes_data.upper),
+            "forecast_lower":float(bikes_data.lower),
+            "DateTime":bikes_data.date_time
+        })
+
+    return data
+
+def change_date_string_to_datetime(date_string):
+    return datetime.datetime.fromisoformat(date_string)
+
+@sync_to_async
+def check_is_entry_today(today):
+    todays_data = Bikes.objects.filter(date_time=today)
+    print('doays_data', todays_data, today)
+    if todays_data:
+        print("todays_data", todays_data[0].__dict__)
+        print("todays_data.exists() and todays_data[0].actual != None", todays_data.exists() and todays_data[0].actual != None)
+        print("todays_data[0].actual != None", todays_data[0].actual != None, "todays_data[0]", todays_data[0].__dict__)
+    return todays_data.exists() and todays_data[0].actual != None