Non-func maintenance

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.4.2"
+    rev: "v0.4.7"
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

htdocs/cscap/dl/dl.py

@@ -10,7 +10,6 @@
 import re
 import shutil
 import smtplib
-import sys
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 
@@ -158,14 +157,6 @@ def replace_varname(varname):
     return varname
 
 
-def pprint(msg):
-    """log a pretty message for my debugging fun"""
-    utcnow = datetime.datetime.utcnow()
-    delta = (utcnow - MEMORY["stamp"]).total_seconds()
-    MEMORY["stamp"] = utcnow
-    sys.stderr.write("timedelta: %.3f %s\n" % (delta, msg))
-
-
 def valid2date(df):
     """If dataframe has valid in columns, rename it to date"""
     if "valid" in df.columns:
@@ -178,7 +169,6 @@ def redup(arr):
     for key in arr:
         if key in AGG:
             additional.extend(AGG[key])
-    pprint("dedup added %s to %s" % (str(additional), str(arr)))
     return arr + additional
 
 
@@ -409,9 +399,7 @@ def do_soil(writer, sites, soil, years, detectlimit, missing):
         },
         index_col=None,
     )
-    pprint("do_soil() query done")
     df["value"] = df["value"].apply(lambda x: conv(x, detectlimit))
-    pprint("do_soil() value replacement done")
     df = pd.pivot_table(
         df,
         index=(
@@ -449,31 +437,12 @@ def do_soil(writer, sites, soil, years, detectlimit, missing):
     df = df.reindex(cols, axis=1)
     # String aggregate above creates a mixture of None and "None"
     df.replace(["None", None], np.nan, inplace=True)
-    pprint(
-        "do_soil() len of inbound df %s"
-        % (
-            len(
-                df.index,
-            )
-        )
-    )
     df.dropna(how="all", inplace=True)
     df.fillna(missing, inplace=True)
-    pprint(
-        "do_soil() len of outbound df %s"
-        % (
-            len(
-                df.index,
-            )
-        )
-    )
-    pprint("do_soil() pivot_table done")
     df.reset_index(inplace=True)
     df["sampledate"] = df["sampledate"].replace("", missing)
     valid2date(df)
-    pprint("do_soil() valid2date done")
     df, worksheet = add_bling(writer, df, "Soil", "Soil")
-    pprint("do_soil() to_excel done")
     workbook = writer.book
     format1 = workbook.add_format({"num_format": "@"})
     worksheet.set_column("B:B", 12, format1)
@@ -702,60 +671,47 @@ def do_work(environ, start_response):
     missing = environ.get("missing", "M")
     if missing == "__custom__":
         missing = environ.get("custom_missing", "M")
-    pprint("Missing is %s" % (missing,))
     detectlimit = environ.get("detectlimit", "1")
 
     writer = pd.ExcelWriter("/tmp/cscap.xlsx", engine="xlsxwriter")
 
     # First sheet is Data Dictionary
     if "SHM5" in shm:
         do_dictionary(writer)
-        pprint("do_dictionary() is done")
 
     # Sheet two is plot IDs
     if "SHM4" in shm:
         do_plotids(writer, sites)
-        pprint("do_plotids() is done")
 
     # Measurement Data
     if agronomic:
         do_agronomic(writer, sites, agronomic, years, detectlimit, missing)
-        pprint("do_agronomic() is done")
     if soil:
         do_soil(writer, sites, soil, years, detectlimit, missing)
-        pprint("do_soil() is done")
     if ghg:
         do_ghg(writer, sites, ghg, years, missing)
-        pprint("do_ghg() is done")
     if ipm:
         do_ipm(writer, sites, ipm, years, missing)
-        pprint("do_ipm() is done")
 
     # Management
     # Field Operations
     if "SHM1" in shm:
         do_operations(writer, sites, years, missing)
-        pprint("do_operations() is done")
     # Pesticides
     if "SHM2" in shm:
         do_pesticides(writer, sites, years)
-        pprint("do_pesticides() is done")
     # Residue and Irrigation
     if "SHM3" in shm:
         do_management(writer, sites, years)
-        pprint("do_management() is done")
     # Site Metadata
     if "SHM8" in shm:
         do_metadata_master(writer, sites, missing)
-        pprint("do_metadata_master() is done")
     # Drainage Management
     if "SHM7" in shm:
         do_dwm(writer, sites, missing)
-        pprint("do_dwm() is done")
     # Notes
     if "SHM6" in shm:
         do_notes(writer, sites, missing)
-        pprint("do_notes() is done")
 
     # Send to client
     writer.close()
@@ -789,7 +745,6 @@ def do_work(environ, start_response):
     )
     cursor.close()
     pgconn.commit()
-    pprint("is done!!!")
     start_response("200 OK", [("Content-type", "text/plain")])
     return [b"Email Delivered!"]
 

htdocs/cscap/dl/filter.py

@@ -9,7 +9,6 @@
 """
 
 import json
-import sys
 
 import pandas as pd
 from pyiem.util import get_dbconnstr
@@ -61,7 +60,6 @@ def redup(arr):
         for val in vals:
             if val in arr and key not in additional:
                 additional.append(key)
-    sys.stderr.write("dedup added %s to %s\n" % (str(additional), str(arr)))
     return arr + additional
 
 
@@ -73,7 +71,6 @@ def agg(arr):
             additional += AGG[val]
     if len(additional) > 0:
         arr += additional
-    sys.stderr.write("agg added %s to %s\n" % (str(additional), str(arr)))
     return arr
 
 

htdocs/cscap/mantable.py

@@ -1,7 +1,6 @@
 """Management Table used by cover crop paper"""
 
 import datetime
-import sys
 
 from pandas.io.sql import read_sql
 from pyiem.util import get_dbconn
@@ -98,13 +97,7 @@ def application(environ, start_response):
             "other",
         ]:
             plantcorndate = _d["plant_corn"]
-            # sys.stderr.write("%s %s %s %s %s\n" % (plantcorndate, valid,
-            #                                       fertilizercrop, site,
-            #                                       cropyear))
             if plantcorndate is None:
-                sys.stderr.write(
-                    ("ERROR! No plant corn for %s %s\n") % (site, cropyear)
-                )
                 continue
             if valid.year < plantcorndate.year:
                 _d[operation + "_fall"] = valid

htdocs/td/dl/dl.py

@@ -9,7 +9,6 @@
 import os
 import shutil
 import smtplib
-import sys
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 
@@ -79,14 +78,6 @@
 MEMORY = dict(stamp=datetime.datetime.utcnow())
 
 
-def pprint(msg):
-    """log a pretty message for my debugging fun"""
-    utcnow = datetime.datetime.utcnow()
-    delta = (utcnow - MEMORY["stamp"]).total_seconds()
-    MEMORY["stamp"] = utcnow
-    sys.stderr.write("timedelta: %.3f %s\n" % (delta, msg))
-
-
 def valid2date(df):
     """If dataframe has valid in columns, rename it to date"""
     if "valid" in df.columns:
@@ -265,7 +256,6 @@ def do_work(form):
     missing = form.get("missing", "M")
     if missing == "__custom__":
         missing = form.get("custom_missing", "M")
-    pprint("Missing is %s" % (missing,))
     # detectlimit = form.get("detectlimit", "1")
 
     # pylint: disable=abstract-class-instantiated
@@ -277,7 +267,6 @@ def do_work(form):
     # First sheet is Data Dictionary
     if "SHM5" in shm or "_ALL" in shm:
         do_dictionary(pgconn, writer)
-        pprint("do_dictionary() is done")
 
     # Sheet two is plot IDs
     if "SHM4" in shm or "_ALL" in shm:
@@ -309,7 +298,6 @@ def do_work(form):
             agronomic,
             missing,
         )
-        pprint("Agronomic is done")
     if water:
         water.extend(["depth", "dwm_treatment", "sample_type", "height"])
         cols = ["soil_moisture", "soil_temperature", "soil_ec"]
@@ -324,7 +312,6 @@ def do_work(form):
                 water,
                 missing,
             )
-            pprint("Soil Moisture is done")
         cols = (
             "tile_flow discharge nitrate_n_load nitrate_n_removed "
             "tile_flow_filled nitrate_n_load_filled"
@@ -340,7 +327,6 @@ def do_work(form):
                 water,
                 missing,
             )
-            pprint("Tile Flow and Loads is done")
         cols = (
             "nitrate_n_concentration ammonia_n_concentration "
             "total_n_filtered_concentration total_n_unfiltered_concentration "
@@ -359,7 +345,6 @@ def do_work(form):
                 water,
                 missing,
             )
-            pprint("Tile Flow and Loads is done")
         cols = [
             "water_table_depth",
         ]
@@ -399,7 +384,6 @@ def do_work(form):
             soil,
             missing,
         )
-        pprint("Soil is done")
 
     # Management
     if "SHM1" in shm or "_ALL" in shm:
@@ -437,7 +421,6 @@ def do_work(form):
         ["_ALL"],
         missing,
     )
-    pprint("do_dwm() is done")
     # Methods
     if "SHM1" in shm or "_ALL" in shm:
         do_generic(
@@ -463,7 +446,6 @@ def do_work(form):
             ["_ALL"],
             missing,
         )
-        pprint("do_notes() is done")
     # Notes
     if "SHM8" in shm or "_ALL" in shm:
         do_generic(
@@ -493,7 +475,6 @@ def do_work(form):
     msg["From"] = "ISU Data Team <[email protected]>"
     msg["To"] = email
     msg.preamble = "Data"
-    pprint(f"Created spreadsheet: /tmp/{tmpfn}")
     try:
         shutil.copyfile(f"/tmp/{tmpfn}", f"/var/webtmp/{tmpfn}")
         os.unlink(f"/tmp/{tmpfn}")
@@ -517,7 +498,6 @@ def do_work(form):
     )
     cursor.close()
     pgconn.commit()
-    pprint("is done!!!")
     return b"Email Delivered!"
 
 

htdocs/td/dl/filter.py

@@ -9,7 +9,6 @@
 """
 
 import json
-import sys
 
 import pandas as pd
 from paste.request import parse_formvars
@@ -61,7 +60,6 @@ def redup(arr):
         for val in vals:
             if val in arr and key not in additional:
                 additional.append(key)
-    sys.stderr.write("dedup added %s to %s\n" % (str(additional), str(arr)))
     return arr + additional
 
 
@@ -75,7 +73,6 @@ def agg(arr):
             additional += AGG[val]
     if len(additional) > 0:
         arr += additional
-    sys.stderr.write("agg added %s to %s\n" % (str(additional), str(arr)))
     return arr
 
 

htdocs/td/plot_agronomic.py

@@ -70,7 +70,6 @@ def make_plot(form, start_response):
         params=(uniqueid,),
         index_col=None,
     )
-    sys.stderr.write(uniqueid)
     if df.empty:
         return send_error(
             start_response, "js", "No / Not Enough Data Found, sorry!"

pyproject.toml

@@ -1,6 +1,11 @@
 
 [tool.ruff]
 line-length = 79
-lint.select = ["E", "F", "I"]
 target-version = "py39"
 
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "I",
+]

src/isudatateam/cscap_utils.py

@@ -5,7 +5,6 @@
 import json
 import os
 import re
-import sys
 
 import smartsheet
 from google.oauth2.service_account import Credentials
@@ -37,9 +36,6 @@ def get_config(filename=None):
     if filename is None:
         filename = CONFIG_FN
     if not os.path.isfile(filename):
-        sys.stderr.write(
-            f"cscap_utils.get_config({filename}) File Not Found.\n"
-        )
         return None
     with open(filename, encoding="utf-8") as fh:
         res = json.load(fh)