Fix and run pre commit (#28)

* Add pyupgrade, follow black's move to psf, update other pre-commit hooks, and remove restriction on pydocstyle < 4 (this fixes #21).

* Run pre-commit hooks on all files, works with few irrelevant exceptions

Author: hmgaudecker

.pre-commit-config.yaml

@@ -2,12 +2,12 @@
 exclude: ^\.
 repos:
 -   repo: https://github.com/asottile/reorder_python_imports
-    rev: v1.4.0
+    rev: v1.7.0
     hooks:
     -   id: reorder-python-imports
         files: '(\.pyi?|wscript)$'
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.1.0
+    rev: v2.3.0
     hooks:
     -   id: check-yaml
     -   id: check-added-large-files
@@ -19,28 +19,35 @@ repos:
     -   id: pretty-format-json
         args: [--autofix, --no-sort-keys]
     -   id: trailing-whitespace
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v1.24.0
+    hooks:
+    -   id: pyupgrade
+        args: [
+          --py36-plus
+        ]
 -   repo: https://github.com/asottile/blacken-docs
-    rev: v0.5.0
+    rev: v1.3.0
     hooks:
     -   id: blacken-docs
         additional_dependencies: [black==19.3b0]
         files: '(\.md|\.rst)$'
--   repo: https://github.com/ambv/black
-    rev: stable
+-   repo: https://github.com/psf/black
+    rev: 19.3b0
     hooks:
     -   id: black
         files: '(\.pyi?|wscript)$'
         language_version: python3.7
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.7
+    rev: 3.7.8
     hooks:
     - id: flake8
       files: '(\.py|wscript)$'
       additional_dependencies: [
         flake8-bugbear, flake8-builtins, flake8-comprehensions,
         flake8-deprecated, flake8-docstrings, flake8-eradicate, flake8-print,
          flake8-rst-docstrings, flake8-todo, flake8-type-annotations,
-        pep8-naming, pydocstyle<4
+        pep8-naming, pydocstyle
       ]
       # Harmonizing flake8 and black
       args: [

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
-filterwarnings = 
+filterwarnings =
 	ignore::DeprecationWarning
 	ignore::PendingDeprecationWarning

src/analysis/__init__.py

@@ -1 +0,0 @@
-

src/analysis/hypo.py

@@ -1,30 +1,31 @@
-# -*- coding: utf-8 -*-
 # !/usr/bin/env python3
 """
 Created on Fri Jun 15 14:36:30 2018
 
 @author: iza6354
 """
+import itertools
+
 import matplotlib.pyplot as plt
-import pandas as pd
 import numpy as np
-import itertools
+import pandas as pd
+from custompygraph.make_plot import make_plot
 from termcolor import cprint
 
-# from check_hypo import check_hypo
-from custompygraph.make_plot import make_plot
 from bld.project_paths import project_paths_join as ppj
-
-from src.model_code.hypo_helpers import (
-    get_ref_text,
-    hypo_graph_settings,
-    get_reform_names,
-    get_hh_text,
-)
-from src.model_code.imports import get_params, say_hello, tarif_ubi
 from src.analysis.tax_transfer import tax_transfer
 from src.analysis.tax_transfer_funcs.taxes import tarif
-from src.analysis.tax_transfer_ubi import tax_transfer_ubi, ubi_settings
+from src.analysis.tax_transfer_ubi import tax_transfer_ubi
+from src.analysis.tax_transfer_ubi import ubi_settings
+from src.model_code.hypo_helpers import get_hh_text
+from src.model_code.hypo_helpers import get_ref_text
+from src.model_code.hypo_helpers import get_reform_names
+from src.model_code.hypo_helpers import hypo_graph_settings
+from src.model_code.imports import get_params
+from src.model_code.imports import say_hello
+from src.model_code.imports import tarif_ubi
+
+# from check_hypo import check_hypo
 
 
 def flip(items, ncol):
@@ -185,7 +186,7 @@ def make_comp_plots(lego, t, maxinc, xlabels, ylabels, lang, settings, ref):
         ncol=ncol,
         frameon=False,
     )
-    plt.savefig(ppj("OUT_FIGURES", "hypo/lego_{}_{}_{}.png".format(ref, t, lang)))
+    plt.savefig(ppj("OUT_FIGURES", f"hypo/lego_{ref}_{t}_{lang}.png"))
 
 
 def create_hypo_data(settings, tb, types, rents, avg_wage=51286):
@@ -509,7 +510,7 @@ def create_hypo_data(settings, tb, types, rents, avg_wage=51286):
 
     df["child6_num"] = df["child3_6_num"]
     for var in ["11", "15", "18", ""]:
-        df["child{}_num".format(var)] = df["child3_6_num"] + df["child7_16_num"]
+        df[f"child{var}_num"] = df["child3_6_num"] + df["child7_16_num"]
 
     # Erster Mann ist immer Head, der Rest sind Frauen und Mädchen
     df["head"] = ~df["female"]
@@ -536,7 +537,7 @@ def create_hypo_data(settings, tb, types, rents, avg_wage=51286):
         "child",
     ]
     for var in tuvars:
-        df["{}num_tu".format(var)] = df["{}_num".format(var)]
+        df[f"{var}num_tu"] = df[f"{var}_num"]
 
     df["hhsize_tu"] = df["hhsize"]
     df["adult_num_tu"] = df["adult_num"]
@@ -695,7 +696,7 @@ def hypo_graphs(dfs, settings, types, lang):
                 ylim_low=0,
                 xlim_low=0,
                 xlim_high=maxinc / 12,
-            ).savefig(ppj("OUT_FIGURES", "hypo/{}_{}_{}.png".format(plottype, t, lang)))
+            ).savefig(ppj("OUT_FIGURES", f"hypo/{plottype}_{t}_{lang}.png"))
 
         # These plots are only for non-baseline reforms
         for plottype in ["budget_diff"]:
@@ -710,7 +711,7 @@ def hypo_graphs(dfs, settings, types, lang):
                 xlim_high=maxinc / 12,
                 showlegend=False,
                 hline=[0],
-            ).savefig(ppj("OUT_FIGURES", "hypo/{}_{}_{}.png".format(plottype, t, lang)))
+            ).savefig(ppj("OUT_FIGURES", f"hypo/{plottype}_{t}_{lang}.png"))
     # Empty memory
     plt.clf()
 
@@ -883,7 +884,7 @@ def hypo_excel(dfs, settings, types):
     # cprint('Producing Excel Output for debugging...', 'red', 'on_white')
     for ref, df in dfs.items():
         df = df.sort_values(by=["typ_bud", "y_wage"])
-        writer = pd.ExcelWriter(ppj("OUT_DATA", "check_hypo_{}.xlsx".format(ref)))
+        writer = pd.ExcelWriter(ppj("OUT_DATA", f"check_hypo_{ref}.xlsx"))
 
         for typ in types:
             df.loc[(df["typ_bud"] == typ)].to_excel(
@@ -981,7 +982,7 @@ def hypo_excel(dfs, settings, types):
 
         # Export to check against Stata Output
         taxout_hypo[ref][taxout_hypo[ref]["head"]].to_json(
-            ppj("OUT_DATA", "python_check_{}.json".format(ref))
+            ppj("OUT_DATA", f"python_check_{ref}.json")
         )
 
     # produce excel control output

src/analysis/meanwages.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
 Created on Fri May 24 09:17:49 2019
 

src/analysis/output.py

@@ -1,12 +1,12 @@
-# -*- coding: utf-8 -*-
 """
 """
-import pandas as pd
-import numpy as np
 import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
 import seaborn as sns
-from src.model_code.hypo_helpers import get_reform_names
+
 from bld.project_paths import project_paths_join as ppj
+from src.model_code.hypo_helpers import get_reform_names
 
 
 def output(settings):
@@ -16,7 +16,7 @@ def output(settings):
         - Benefit recipients
 
     """
-  
+
     budget = pd.DataFrame(columns=settings["Reforms"])
     recip = pd.DataFrame(columns=settings["Reforms"])
     dpis = pd.DataFrame(columns=settings["Reforms"])
@@ -29,7 +29,7 @@ def output(settings):
         budgetvars = taxvars + benvars
 
         # load reform-specific results
-        df = pd.read_json(ppj("OUT_DATA", "taxben_results_{}.json".format(ref)))
+        df = pd.read_json(ppj("OUT_DATA", f"taxben_results_{ref}.json"))
         # print("Number of adults: {}".format(df[df['age']>=18]['pweight'].sum()))
         # print("Number of children: {}".format(df[df['age']<18]['pweight'].sum()))
 
@@ -53,19 +53,19 @@ def output(settings):
 
         # also calculate equivalized income
         df["eq_scale"] = (
-                1 + 0.5 * np.maximum((df["hhsize"] - df["child14_num"] - 1), 0) + 0.3 * (df["child14_num"])
+            1
+            + 0.5 * np.maximum((df["hhsize"] - df["child14_num"] - 1), 0)
+            + 0.3 * (df["child14_num"])
         )
 
         dpis[ref]["dpi_eq"] = df["dpi"] / df["eq_scale"]
         dpis[ref]["dpi_per_head"] = df["dpi"] / df["hhsize"]
-        dpis[ref]["dpi"]    = df["dpi"]
+        dpis[ref]["dpi"] = df["dpi"]
         dpis[ref]["pweight"] = df["pweight"]
         if ref != base:
             dpis[ref]["d_dpi"] = dpis[ref]["dpi_per_head"] - dpis[base]["dpi_per_head"]
             # print(dpis[ref]["d_dpi"].describe())
 
-
-
     # calculate total budget
     budget.loc["TOTAL"] = budget.sum()
     # Calculate Differences to baseline
@@ -78,7 +78,6 @@ def output(settings):
             diff_rev[ref] = budget[ref] - budget[base]
             recip_rev[ref] = recip[ref] - recip[base]
 
-
     print("-" * 80)
     print("Budget Differences (bn € per year)")
     print(diff_rev)
@@ -89,54 +88,68 @@ def output(settings):
     # TODO: output to HD
     # DISTRIBUTIONAL OUTPUT
     # Density Plots. Note that these are unweighted!
-    fig = plt.figure(figsize=(8,5))
+    fig = plt.figure(figsize=(8, 5))
     for ref in settings["Reforms"]:
-        sns.distplot(dpis[ref]["dpi_eq"][dpis[ref]["dpi_eq"].between(0,5000)],
-                     kde=True,
-                     hist=False,
-                     kde_kws={'shade': True,
-                              'bw': 100},
-                     label=get_reform_names("en")[ref]
-                     )
-    plt.title('Income Distributions')
-    plt.xlabel('Personal Income')
-    plt.ylabel('Density')
+        sns.distplot(
+            dpis[ref]["dpi_eq"][dpis[ref]["dpi_eq"].between(0, 5000)],
+            kde=True,
+            hist=False,
+            kde_kws={"shade": True, "bw": 100},
+            label=get_reform_names("en")[ref],
+        )
+    plt.title("Income Distributions")
+    plt.xlabel("Personal Income")
+    plt.ylabel("Density")
     plt.savefig(ppj("OUT_FIGURES", "income_densities.png"))
 
     # Winner/Loser Analysis
     for ref in settings["Reforms"][1:]:
-        dpis[ref]["winner"] = df["pweight"] * (dpis[ref]["dpi"] > (dpis[base]["dpi"] + 5))
-        dpis[ref]["loser"]  = df["pweight"] * (dpis[ref]["dpi"] < (dpis[base]["dpi"] - 5))
+        dpis[ref]["winner"] = df["pweight"] * (
+            dpis[ref]["dpi"] > (dpis[base]["dpi"] + 5)
+        )
+        dpis[ref]["loser"] = df["pweight"] * (
+            dpis[ref]["dpi"] < (dpis[base]["dpi"] - 5)
+        )
         winshare = 100 * (dpis[ref]["winner"].sum() / dpis[ref]["pweight"].sum())
         loseshare = 100 * (dpis[ref]["loser"].sum() / dpis[ref]["pweight"].sum())
         winavg = dpis[ref]["d_dpi"][dpis[ref]["winner"] > 0].mean()
         losavg = dpis[ref]["d_dpi"][dpis[ref]["loser"] > 0].mean()
-        print("Income Gains: {}".format(dpis[ref]["d_dpi"][dpis[ref]["winner"] > 0].describe()))
-        print("Income Losses: {}".format(dpis[ref]["d_dpi"][dpis[ref]["loser"] > 0].describe()))
-
-        fig = plt.figure(figsize=(8,5))
-        sns.distplot(dpis[ref]["d_dpi"][dpis[ref]["d_dpi"].between(-5000,2000)],
-                     kde=True,
-                     hist=False,
-                     kde_kws={'shade': True,
-                              'bw': 100},
-                     )
-#        sns.distplot(dpis[ref]["d_dpi"][dpis[ref]["d_dpi"].between(.01,2000)],
-#                     kde=True,
-#                     hist=False,
-#                     kde_kws={'shade': True,
-#                              'bw': 100},
-#                     label="Winners"
-#                     )
-        plt.text(-2000, .0004, "{:.2f}% Losers \nAverage Loss: € {:.0f}".format(loseshare,
-                         losavg * (-1)))
-        plt.text(500, .0006, "{:.2f}% Winners \nAverage Gain: € {:.0f}".format(winshare,
-                 winavg))
+        print(
+            "Income Gains: {}".format(
+                dpis[ref]["d_dpi"][dpis[ref]["winner"] > 0].describe()
+            )
+        )
+        print(
+            "Income Losses: {}".format(
+                dpis[ref]["d_dpi"][dpis[ref]["loser"] > 0].describe()
+            )
+        )
+
+        fig = plt.figure(figsize=(8, 5))
+        sns.distplot(
+            dpis[ref]["d_dpi"][dpis[ref]["d_dpi"].between(-5000, 2000)],
+            kde=True,
+            hist=False,
+            kde_kws={"shade": True, "bw": 100},
+        )
+        #        sns.distplot(dpis[ref]["d_dpi"][dpis[ref]["d_dpi"].between(.01,2000)],
+        #                     kde=True,
+        #                     hist=False,
+        #                     kde_kws={'shade': True,
+        #                              'bw': 100},
+        #                     label="Winners"
+        #                     )
+        plt.text(
+            -2000,
+            0.0004,
+            "{:.2f}% Losers \nAverage Loss: € {:.0f}".format(loseshare, losavg * (-1)),
+        )
+        plt.text(500, 0.0006, f"{winshare:.2f}% Winners \nAverage Gain: € {winavg:.0f}")
         plt.axvline(0)
-        plt.title('Distribution of income change')
-        plt.xlabel('Monthly income per person')
-        plt.ylabel('Density')
-        plt.savefig(ppj("OUT_FIGURES", "d_inc_per_person_{}.png".format(ref)))
+        plt.title("Distribution of income change")
+        plt.xlabel("Monthly income per person")
+        plt.ylabel("Density")
+        plt.savefig(ppj("OUT_FIGURES", f"d_inc_per_person_{ref}.png"))
 
 
 #    print(df["dpi_eq"].describe())

src/analysis/tax_transfer.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
 TAX TRANSFER SIMULATION
 

src/analysis/tax_transfer_funcs/benefits/wohngeld.py

@@ -122,7 +122,7 @@ def calc_wg_income(df, tb, hhsize):
         "gkvbeit",
         "uhv",
     ]:
-        wg_income["{}_tu_k".format(inc)] = aggr(df, inc, "all_tu")
+        wg_income[f"{inc}_tu_k"] = aggr(df, inc, "all_tu")
 
     wg_income["wg_abzuege"] = calc_wg_abzuege(wg_income, tb)
 

src/analysis/tax_transfer_funcs/taxes/calc_taxes.py

@@ -19,10 +19,8 @@ def tax_sched(df, tb):
     ts = pd.DataFrame(index=df.index.copy())
     for inc in tb["zve_list"]:
         ts["tax_" + inc] = tb["tax_schedule"](df["zve_" + inc], tb)
-        ts["tax_{}_tu".format(inc)] = ts["tax_{}".format(inc)]
-        ts.loc[adult_married, "tax_{}_tu".format(inc)] = ts["tax_{}".format(inc)][
-            adult_married
-        ].sum()
+        ts[f"tax_{inc}_tu"] = ts[f"tax_{inc}"]
+        ts.loc[adult_married, f"tax_{inc}_tu"] = ts[f"tax_{inc}"][adult_married].sum()
 
     # Abgeltungssteuer
     ts["abgst"] = abgeltung(df, tb)
@@ -49,8 +47,8 @@ def tax_sched(df, tb):
         [df["zveranl"], ~df["zveranl"]], [ts["soli_tu"] / 2, ts["soli_tu"]]
     )
     return ts[
-        ["tax_{}".format(inc) for inc in tb["zve_list"]]
-        + ["tax_{}_tu".format(inc) for inc in tb["zve_list"]]
+        [f"tax_{inc}" for inc in tb["zve_list"]]
+        + [f"tax_{inc}_tu" for inc in tb["zve_list"]]
         + ["abgst_tu", "abgst", "soli", "soli_tu"]
     ]