Merge pull request #186 from saharsh-agrawal/caching

Caching done correctly (atleast locally)

Author: Devansh-bit

.gitignore

@@ -13,7 +13,7 @@ venv
 
 *.pdf
 Academic_Cal-j/**
-final.json
+cache/**
 
 # auto login
 erpcreds.py

Dockerfile

@@ -20,6 +20,6 @@ RUN chmod +x ./postinstall.sh
 
 COPY . .
 
-RUN python download-calendar.py
+# RUN python download-calendar.py
 
 CMD [ "./postinstall.sh", "gunicorn", "--bind", "0.0.0.0:8000", "wsgi:app" ]

app.py

@@ -217,14 +217,17 @@ def download_ics():
 
         # Create an in-memory file-like object for the ics content
         ics_file = io.BytesIO()
-        ics_file.write(ics_content.encode("utf-8"))
+        # generate_ics returns bytes already; write directly
+        if isinstance(ics_content, str):
+            ics_content = ics_content.encode("utf-8")
+        ics_file.write(ics_content)
         ics_file.seek(0)
 
         return send_file(
             ics_file,
             as_attachment=True,
             mimetype="text/calendar",
-            download_name=f"${roll_number}-timetable.ics",
+            download_name=f"{roll_number}-timetable.ics",
         )
     except Exception as e:
         return jsonify({"status": "error", "message": str(e)}), 500
@@ -260,7 +263,9 @@ def image_parser():
 
         # Create an in-memory file-like object for the ics content
         ics_file = io.BytesIO()
-        ics_file.write(ics_content.encode("utf-8"))
+        if isinstance(ics_content, str):
+            ics_content = ics_content.encode("utf-8")
+        ics_file.write(ics_content)
         ics_file.seek(0)
 
         return send_file(

timetable/generate_ics.py

@@ -4,6 +4,7 @@
 from datetime import timedelta
 from timetable import Course
 from utils import academic_calander_handler, dates, build_event_duration, generate_india_time
+from utils.holidays_handler import get_holidays
 
 
 def generate_ics(courses: list[Course], output_filename, is_web=False):
@@ -60,7 +61,7 @@ def first_occurrence_of_day(day_name: str):
         cal.add_component(event)
 
     # add holidays (as all-day events in Asia/Kolkata)
-    for holiday in dates.holidays:
+    for holiday in get_holidays():
         event = Event()
         event.add("summary", "INSTITUTE HOLIDAY : " + holiday[0])
         hdt = holiday[1]

timetable/google_calendar.py

@@ -12,10 +12,10 @@
 from utils import (
     END_TERM_BEGIN,
     SEM_BEGIN,
-    holidays,
     dates,
     generate_india_time,
 )
+from utils.holidays_handler import get_holidays
 from utils import academic_calander_handler
 from timetable import Course
 
@@ -132,7 +132,7 @@ def first_occurrence_of_day(day_name: str):
     batch.execute()  ## execute batch of timetable
 
     # add holidays to calendar as all-day events (Asia/Kolkata midnight)
-    for holiday in holidays:
+    for holiday in get_holidays():
         hdt = holiday[1]
         start_str = generate_india_time(hdt.year, hdt.month, hdt.day, 0, 0).strftime("%Y-%m-%dT00:00:00")
         end_dt = generate_india_time(hdt.year, hdt.month, hdt.day, 0, 0) + timedelta(days=1)
@@ -148,6 +148,7 @@ def first_occurrence_of_day(day_name: str):
             },
         }
         service.events().insert(calendarId=calendar_id, body=holiday_event).execute()
+    print("Added holidays")
 
     # add academic calendar entries as all-day events
     for entry in academic_calander_handler.get_academic_calendar(is_web):
@@ -159,6 +160,7 @@ def first_occurrence_of_day(day_name: str):
             "end": {"dateTime": end_str, "timeZone": "Asia/Kolkata"},
         }
         service.events().insert(calendarId=calendar_id, body=event).execute()
+    print("Added academic calendar entries")
 
     print("\nAll events added successfully!\n")
 

utils/academic_calander_handler.py

@@ -1,4 +1,3 @@
-import tempfile
 from datetime import datetime, timedelta
 import glob
 import camelot
@@ -11,6 +10,7 @@
 import re
 
 JSON_FOLDER_NAME = 'Academic_Cal-j'
+CACHE_DIR = 'cache'
 
 
 @dataclass
@@ -37,6 +37,18 @@ def get_latest_calendar_name():
     return filename
 
 
+def ensure_cache_dir():
+    if not os.path.isdir(CACHE_DIR):
+        os.makedirs(CACHE_DIR, exist_ok=True)
+
+
+def get_cache_path():
+    filename = get_latest_calendar_name()
+    stem = os.path.splitext(filename)[0]
+    ensure_cache_dir()
+    return os.path.join(CACHE_DIR, f"{stem}.json")
+
+
 def is_file_present(file):
     if (os.path.exists(cwd() + '/' + file) or
             os.path.exists(cwd() + '/' + file + '/')
@@ -77,7 +89,7 @@ def get_latest_calendar(is_web=False):
 
     try:
         with open(filename, "wb") as file:
-            response = requests.get(url)
+            response = requests.get(url, timeout=15)
             response.raise_for_status()
             file.write(response.content)
     except Exception as e:
@@ -129,7 +141,7 @@ def export_json():
 def get_json_files():
     folder_path = cwd() + '/' + JSON_FOLDER_NAME
     if (is_file_present(JSON_FOLDER_NAME)):
-        files = glob.glob(folder_path + '/*.json', include_hidden=True)
+        files = glob.glob(folder_path + '/*.json')
         return files
     else:
         return []
@@ -141,27 +153,37 @@ def merge_json():
         with open(file) as f:
             data = json.load(f)
             merged_data.extend(data)
-
-    with open('final.json', "w") as f:
-        json.dump(merged_data, f, indent=4)
-
     return merged_data
 
+def cleanup_artifacts():
+    """Remove intermediate artifacts: extracted JSON folder, zip, and PDF."""
+    try:
+        delete_file(JSON_FOLDER_NAME)
+    except Exception:
+        pass
+    try:
+        delete_file(JSON_FOLDER_NAME + '.zip')
+    except Exception:
+        pass
+    try:
+        delete_file(get_latest_calendar_name())
+    except Exception:
+        pass
 
-def clean_temp_files():
-    base = tempfile.gettempdir()
-    for filename in os.listdir(base):
-        if not filename.startswith('tmp') or len(filename) != 11:
-            continue
-        fullpath = os.path.join(base, filename)
+def get_academic_calendar(is_web = False) -> list[DataEntry]:
+    # Try cache first
+    cache_path = get_cache_path()
+    if os.path.isfile(cache_path):
         try:
-            shutil.rmtree(fullpath)
-        except Exception as E:
-            print(E)
-            continue
+            with open(cache_path, 'r', encoding='utf-8') as f:
+                cached = json.load(f)
+                entries = [DataEntry(start_date=datetime.fromisoformat(x['start_date']),
+                                     end_date=datetime.fromisoformat(x['end_date']),
+                                     event=x['event']) for x in cached]
+            return entries
+        except Exception:
+            pass
 
-
-def get_academic_calendar(is_web = False) -> list[DataEntry]:
     get_latest_calendar(is_web)
     export_json()
 
@@ -223,13 +245,20 @@ def get_academic_calendar(is_web = False) -> list[DataEntry]:
         if (len(annual_convocation) == 2 and ("annual" in annual_convocation or "convocation" in annual_convocation)):
             break
 
-    ## This has to be done to remove temporary files created by camelot. These files are not automatically
-    ## deleted until program exits
-    ## This is not ideal, and might be dangerous (and invisible) if other programs are creating similar directories often
-    ## Nothing else can be done without modifying `camelot`.
+    # Cache for subsequent runs
     try:
-        clean_temp_files()
-    except Exception as E:
-        print(E)
+        ensure_cache_dir()
+        with open(cache_path, 'w', encoding='utf-8') as f:
+            json.dump([
+                {
+                    'start_date': e.start_date.isoformat(),
+                    'end_date': e.end_date.isoformat(),
+                    'event': e.event,
+                } for e in main_dates
+            ], f, indent=2)
+    except Exception:
+        pass
+    # After caching, remove intermediate artifacts
+    cleanup_artifacts()
 
     return main_dates

utils/dates.py

@@ -1,9 +1,8 @@
 from __future__ import print_function
-import requests
 from datetime import datetime, timedelta, date
-from bs4 import BeautifulSoup as bs
-from utils import build_event
 import sys
+from utils import build_event
+from utils.holidays_handler import get_holidays
 
 
 SEM_BEGIN = build_event.generate_india_time(2025, 7, 16, 0, 0)
@@ -14,36 +13,6 @@
 AUT_BREAK_END = build_event.generate_india_time(2025, 10, 5, 0, 0)
 
 
-### getting holidays
-def get_holidays() -> tuple[list[tuple[str, datetime]], dict[str, list[datetime]]]:
-    """
-    scrapes holiday list from IITKGP website
-    returns: list of holidays as occasions and datetime objects
-    """
-    url = "https://www.iitkgp.ac.in/holidays"
-    result = requests.get(url).text
-    doc = bs(result, "html.parser")
-    tbody = doc.tbody
-    trs = tbody.contents
-    holidays = []
-    for i in range(3, len(trs) - 7, 2):
-        cnt = 0
-        for tr in trs[i]:
-            cnt = cnt + 1
-            if cnt == 2:
-                occasion = tr.string
-            if cnt == 4:
-                datetime_str = tr.string
-                d = (int)(datetime_str[:2])
-                m = (int)(datetime_str[3:5])
-                y = (int)(datetime_str[6:])
-                hol_date = build_event.generate_india_time(y, m, d, 0, 0)
-                holidays.append([occasion, hol_date])
-
-    holidays.sort(key=lambda x: x[1])
-    return holidays
-
-
 def daterange(start_dt: datetime, end_dt: datetime):
     """
     Yield all dates d such that start_dt.date() <= d.date() <= end_dt.date().
@@ -64,6 +33,9 @@ def get_class_off_dates_in_semester() -> list[datetime]:
       - Entire autumn break range [AUT_BREAK_BEGIN, AUT_BREAK_END]
     Returns tz-aware datetimes at 00:00 Asia/Kolkata for each day off.
     """
+    # Fetch holidays
+    holidays = get_holidays()
+
     # Build from the scraped holidays and fixed ranges
     off = set()
     # Holidays that fall within the semester window (exclude artificial boundary markers)
@@ -108,8 +80,6 @@ def get_class_off_dates_in_semester() -> list[datetime]:
     print("Note: SEM_BEGIN < MID_TERM_BEGIN < MID_TERM_END < END_TERM_BEGIN")
     sys.exit(1)
 
-holidays = get_holidays()
-
 def next_weekday(current_day: datetime, weekday: str) -> datetime:
     days = {
         "Monday": 0,

utils/holidays_handler.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import os
+import json
+from datetime import datetime
+from typing import List, Tuple
+
+import requests
+from bs4 import BeautifulSoup as bs
+
+from utils.build_event import generate_india_time
+
+
+CACHE_DIR = 'cache'
+
+
+def ensure_cache_dir():
+    if not os.path.isdir(CACHE_DIR):
+        os.makedirs(CACHE_DIR, exist_ok=True)
+
+
+def holidays_cache_path() -> str:
+    ensure_cache_dir()
+    year = datetime.today().year
+    return os.path.join(CACHE_DIR, f"holidays_{year}.json")
+
+
+def get_holidays(refresh: bool = False) -> List[Tuple[str, datetime]]:
+    """
+    Fetch IITKGP institute holidays with caching.
+
+    Returns a list of tuples: (occasion, datetime_at_midnight_Asia_Kolkata)
+    """
+    cache_path = holidays_cache_path()
+
+    if not refresh and os.path.isfile(cache_path):
+        try:
+            with open(cache_path, 'r', encoding='utf-8') as f:
+                raw = json.load(f)
+                return [(r[0], generate_india_time(*r[1])) for r in raw]
+        except Exception:
+            pass
+
+    url = "https://www.iitkgp.ac.in/holidays"
+    result = requests.get(url, timeout=15).text
+    doc = bs(result, "html.parser")
+    tbody = doc.tbody
+    trs = tbody.contents
+    holidays: List[Tuple[str, datetime]] = []
+
+    for i in range(3, len(trs) - 7, 2):
+        cnt = 0
+        occasion = None
+        for tr in trs[i]:
+            cnt += 1
+            if cnt == 2:
+                occasion = tr.string
+            if cnt == 4:
+                datetime_str = tr.string
+                try:
+                    dt = datetime.strptime(datetime_str, "%d.%m.%Y")
+                    hol_date = generate_india_time(dt.year, dt.month, dt.day, 0, 0)
+                    holidays.append((occasion, hol_date))
+                except Exception:
+                    continue
+
+    holidays.sort(key=lambda x: x[1])
+
+    # Cache as [[occasion, [year, month, day, hour, minute]], ...]
+    try:
+        with open(cache_path, 'w', encoding='utf-8') as f:
+            json.dump([[h[0], [h[1].year, h[1].month, h[1].day, 0, 0]] for h in holidays], f, indent=2)
+    except Exception:
+        pass
+
+    return holidays