Adiciona spider que coleta dados de casos do Ceará

Author: Cesar Smaniotto

collect.sh

@@ -10,7 +10,7 @@ fi
 source $SCRIPT_PATH/base.sh
 
 mkdir -p $DOWNLOAD_PATH $OUTPUT_PATH $LOG_PATH
-for state in pr; do
+for state in ce pr; do
 	log_filename="$LOG_PATH/caso-${state}.log"
 	csv_filename="$OUTPUT_PATH/caso-${state}.csv"
 	rm -rf "$log_filename" "$csv_filename"

corona_ce_spider.py

@@ -0,0 +1,139 @@
+import json
+from collections import defaultdict
+from datetime import datetime, timedelta
+
+import scrapy
+
+
+class Covid19CESpider(scrapy.Spider):
+    name = "covid19ce"
+    base_url = "https://indicadores.integrasus.saude.ce.gov.br/api/coronavirus/qtd-por-municipio?data={date}&tipo={type}"
+
+    def start_requests(self):
+        yield scrapy.Request(
+            "https://indicadores.integrasus.saude.ce.gov.br/api/coronavirus/filtro-data",
+            self.parse_filter_date,
+        )
+
+    def parse_filter_date(self, response):
+        response_json = response.body_as_unicode()
+        filter_date = json.loads(response_json)[0]
+
+        iter_date = datetime.strptime(filter_date["dataMin"], "%d/%m/%Y").date()
+        end_date = datetime.strptime(filter_date["dataMax"], "%d/%m/%Y").date()
+
+        while iter_date <= end_date:
+            yield scrapy.Request(
+                self.base_url.format(date=iter_date.isoformat(), type="Confirmado"),
+                self.parse_confirmed,
+                meta={"date": iter_date.isoformat()},
+            )
+
+            iter_date += timedelta(days=1)
+
+    def parse_confirmed(self, response):
+        response_json = response.body_as_unicode()
+        confirmed_cases = json.loads(response_json)
+
+        fixed_cases = []
+        for case in confirmed_cases:
+            if case["tipo"] != "Positivo":
+                continue
+
+            fixed_cases.append(
+                {
+                    **case,
+                    "date": response.meta["date"],
+                    "url": response.url,
+                    "confirmed": case["quantidade"],
+                }
+            )
+
+        yield scrapy.Request(
+            self.base_url.format(date=response.meta["date"], type="Óbito"),
+            self.parse_death,
+            meta={"confirmed": fixed_cases, "date": response.meta["date"]},
+        )
+
+    def parse_death(self, response):
+        response_json = response.body_as_unicode()
+        death_cases = json.loads(response_json)
+
+        fixed_cases = []
+        for case in death_cases:
+            if case["tipo"] != "Positivo":
+                continue
+
+            fixed_cases.append(
+                {
+                    **case,
+                    "date": response.meta["date"],
+                    "url": response.url,
+                    "deaths": case["quantidade"],
+                }
+            )
+
+        all_cases = fixed_cases + response.meta["confirmed"]
+        parsed_cases = list(process_cities(all_cases))
+
+        for case in parsed_cases:
+            yield case
+
+        state_case = process_state(parsed_cases)
+        if state_case:
+            yield state_case
+
+
+def process_cities(cases):
+    map_city_case = defaultdict(lambda: {"deaths": 0, "confirmed": 0, "source_url": []})
+
+    for case in cases:
+        municipio = case["municipio"]
+        if "confirmed" in case:
+            map_city_case[municipio]["confirmed"] = case["confirmed"]
+        if "deaths" in case:
+            map_city_case[municipio]["deaths"] = case["deaths"]
+
+        map_city_case[municipio]["source_url"].append(case["url"])
+        map_city_case[municipio].update(
+            date=case["date"], city=case["municipio"].title(),
+        )
+
+    for case in map_city_case.values():
+        yield {
+            "date": case["date"],
+            "state": "CE",
+            "city": case["city"],
+            "place_type": "city",
+            "notified": "",
+            "confirmed": case["confirmed"],
+            "discarded": "",
+            "suspect": "",
+            "deaths": case["deaths"],
+            "notes": "",
+            "source_url": ",".join(case["source_url"]),
+        }
+
+
+def process_state(cities_cases):
+    if not cities_cases:
+        return None
+
+    sum_confirmed = sum((case["confirmed"] for case in cities_cases))
+    sum_deaths = sum((case["deaths"] for case in cities_cases))
+
+    city_case = cities_cases[0]
+
+    return {
+        "date": city_case["date"],
+        "state": "CE",
+        "city": "",
+        "place_type": "state",
+        "notified": "",
+        "confirmed": sum_confirmed,
+        "discarded": "",
+        "suspect": "",
+        "deaths": sum_deaths,
+        "notes": "",
+        "source_url": city_case["source_url"],
+    }