From 13de9930d8b41ddcea9e147ceeb91ae123e17d10 Mon Sep 17 00:00:00 2001 From: Joshua Greben Date: Mon, 11 Nov 2024 07:45:02 -0800 Subject: [PATCH 1/7] Log errors but do not raise SystemExit for encumbrances script --- .../folio/encumbrances/fix_encumbrances.py | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py index 0a993350..b520596b 100644 --- a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py +++ b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py @@ -56,8 +56,8 @@ def login(tenant, username, password): 'Content-Type': 'application/json', } except Exception as err: - logger.info('Error during login:', err) - raise SystemExit(1) + logger.error('Error during login:', err) + # raise SystemExit(1) async def get_request_without_query(url: str) -> dict: @@ -67,11 +67,11 @@ async def get_request_without_query(url: str) -> dict: if resp.status_code == HTTPStatus.OK: return resp.json() else: - logger.info(f'Error getting record with url {url} : \n{resp.text} ') - raise SystemExit(1) + logger.error(f'Error getting record with url {url} : \n{resp.text} ') + # raise SystemExit(1) except Exception as err: - logger.info(f'Error getting record with url {url} : {err=}') - raise SystemExit(1) + logger.error(f'Error getting record with url {url} : {err=}') + # raise SystemExit(1) async def get_request(url: str, query: str) -> dict: @@ -85,11 +85,11 @@ async def get_request(url: str, query: str) -> dict: if resp.status_code == HTTPStatus.OK: return resp.json() else: - logger.info(f'Error getting records by {url} ?query= "{query}": \n{resp.text} ') - raise SystemExit(1) + logger.error(f'Error getting records by {url} ?query= "{query}": \n{resp.text} ') + # raise SystemExit(1) except Exception as err: - logger.info(f'Error getting records by {url}?query={query}: {err=}') - raise SystemExit(1) + logger.error(f'Error getting records by {url}?query={query}: {err=}') + # raise SystemExit(1) async def put_request(url: str, data): @@ -101,12 +101,12 @@ async def put_request(url: str, data): ) if resp.status_code == HTTPStatus.NO_CONTENT: return - logger.info(f'Error updating record {url} "{data}": {resp.text}') - raise SystemExit(1) + logger.error(f'Error updating record {url} "{data}": {resp.text}') + # raise SystemExit(1) except Exception as err: - logger.info(f'Error updating record {url} "{data}": {err=}') - raise SystemExit(1) + logger.error(f'Error updating record {url} "{data}": {err=}') + # raise SystemExit(1) async def delete_request(url: str): @@ -116,12 +116,12 @@ async def delete_request(url: str): resp = await client.delete(url, headers=headers, timeout=ASYNC_CLIENT_TIMEOUT) if resp.status_code == HTTPStatus.NO_CONTENT: return - logger.info(f'Error deleting record {url}: {resp.text}') - raise SystemExit(1) + logger.error(f'Error deleting record {url}: {resp.text}') + # raise SystemExit(1) except Exception as err: - logger.info(f'Error deleting record {url}: {err=}') - raise SystemExit(1) + logger.error(f'Error deleting record {url}: {err=}') + # raise SystemExit(1) def get_fiscal_years_by_query(query) -> dict: @@ -134,8 +134,8 @@ def get_fiscal_years_by_query(query) -> dict: raise_exception_for_reply(r) return r.json()['fiscalYears'] except Exception as err: - logger.info(f'Error getting fiscal years with query "{query}": {err}') - raise SystemExit(1) + logger.error(f'Error getting fiscal years with query "{query}": {err}') + # raise SystemExit(1) def get_by_chunks(url, query, key) -> list: @@ -173,8 +173,8 @@ def get_order_ids_by_query(query) -> list: for order in orders: ids.append(order['id']) except Exception as err: - logger.info(f'Error getting order ids with query "{query}": {err}') - raise SystemExit(1) + logger.error(f'Error getting order ids with query "{query}": {err}') + # raise SystemExit(1) return ids @@ -200,8 +200,8 @@ def get_fiscal_year(fiscal_year_code) -> dict: query = f'code=="{fiscal_year_code}"' fiscal_years = get_fiscal_years_by_query(query) if len(fiscal_years) == 0: - logger.info(f'Could not find fiscal year "{fiscal_year_code}".') - raise SystemExit(1) + logger.error(f'Could not find fiscal year "{fiscal_year_code}".') + # raise SystemExit(1) return fiscal_years[0] @@ -243,10 +243,10 @@ async def get_budget_by_fund_id(fund_id, fiscal_year_id) -> dict: query = f'fundId=={fund_id} AND fiscalYearId=={fiscal_year_id}' budgets = await get_budgets_by_query(query) if len(budgets) == 0: - logger.info( + logger.error( f'Could not find budget for fund "{fund_id}" and fiscal year "{fiscal_year_id}".' ) - raise SystemExit(1) + # raise SystemExit(1) return budgets[0] @@ -419,13 +419,13 @@ async def fix_fund_id_with_duplicate_encumbrances(encumbrances, fd_fund_id, poli else: encumbrances_with_bad_fund.append(encumbrance) if len(encumbrances_with_bad_fund) == 0: - logger.info( + logger.warning( f" Warning: there is a remaining duplicate encumbrance for poline {poline['id']} " f"({poline['poLineNumber']})." ) return if len(encumbrances_with_right_fund) != 1: - logger.info( + logger.warning( f" Problem fixing encumbrances for poline {poline['id']} ({poline['poLineNumber']}), " "please fix by hand." ) @@ -618,10 +618,10 @@ async def fix_order_status_and_release_encumbrances(order_id, encumbrances): await put_request(url, encumbrance) except Exception as err: - logger.info( + logger.error( f'Error when fixing order status in encumbrances for order {order_id}:', err ) - raise SystemExit(1) + # raise SystemExit(1) async def fix_order_encumbrances_order_status(order_id, encumbrances): @@ -642,10 +642,10 @@ async def fix_order_encumbrances_order_status(order_id, encumbrances): order_id, modified_encumbrances ) except Exception as err: - logger.info( + logger.error( f'Error when fixing order status in encumbrances for order {order_id}:', err ) - raise SystemExit(1) + # raise SystemExit(1) async def fix_encumbrance_order_status_for_closed_order( From 1447cdfc6d589e8ce197f3dc286e173ee852a3b4 Mon Sep 17 00:00:00 2001 From: Joshua Greben Date: Mon, 11 Nov 2024 12:58:44 -0800 Subject: [PATCH 2/7] Output logger to file --- .../folio/encumbrances/fix_encumbrances_run.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances_run.py b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances_run.py index 1904f515..e68f4de6 100644 --- a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances_run.py +++ b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances_run.py @@ -1,5 +1,6 @@ import asyncio import contextlib +import logging import pathlib import libsys_airflow.plugins.folio.encumbrances.fix_encumbrances as fix_encumbrances_script @@ -18,8 +19,10 @@ def fix_encumbrances_run(*args, **kwargs): log_path = pathlib.Path(airflow) / f"fix_encumbrances/{library}-{run_id}.log" log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("w+", 1) as log: - with contextlib.redirect_stdout(log): + logging.basicConfig(level=logging.INFO, filename=log) + with contextlib.redirect_stdout(OutputLogger()): asyncio.run( fix_encumbrances_script.run_operation( int(choice), fiscal_year_code, tenant, username, password @@ -27,3 +30,16 @@ def fix_encumbrances_run(*args, **kwargs): ) return str(log_path.absolute()) + + +class OutputLogger: + def __init__(self, name="root", level="INFO"): + self.logger = logging.getLogger(name) + self.level = getattr(logging, level) + + def write(self, msg): + if msg and not msg.isspace(): + self.logger.log(self.level, msg) + + def flush(self): + pass From f99d18c81429c42a6d93f72f236a4f63fd0e3d6c Mon Sep 17 00:00:00 2001 From: Joshua Greben Date: Mon, 11 Nov 2024 13:00:50 -0800 Subject: [PATCH 3/7] Lint --- .../plugins/folio/encumbrances/fix_encumbrances_run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances_run.py b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances_run.py index e68f4de6..415fa3ce 100644 --- a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances_run.py +++ b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances_run.py @@ -19,7 +19,6 @@ def fix_encumbrances_run(*args, **kwargs): log_path = pathlib.Path(airflow) / f"fix_encumbrances/{library}-{run_id}.log" log_path.parent.mkdir(parents=True, exist_ok=True) - with log_path.open("w+", 1) as log: logging.basicConfig(level=logging.INFO, filename=log) with contextlib.redirect_stdout(OutputLogger()): From 0a8112f7a29fe21a4a8d1509502ba9de2bcb71c0 Mon Sep 17 00:00:00 2001 From: Joshua Greben Date: Mon, 11 Nov 2024 13:44:52 -0800 Subject: [PATCH 4/7] Catch logging TypeError --- .../plugins/folio/encumbrances/fix_encumbrances.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py index b520596b..2169cba5 100644 --- a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py +++ b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py @@ -216,7 +216,10 @@ def get_closed_orders_ids() -> list: logger.info('Retrieving closed order ids...') query = 'workflowStatus=="Closed"' closed_orders_ids = get_order_ids_by_query(query) - logger.info(' Closed orders:', len(closed_orders_ids)) + try + logger.info(' Closed orders:', len(closed_orders_ids)) + except TypeError + logger.info('No closed orders') return closed_orders_ids @@ -224,7 +227,10 @@ def get_open_orders_ids() -> list: logger.info('Retrieving open order ids...') query = 'workflowStatus=="Open"' open_orders_ids = get_order_ids_by_query(query) - logger.info(' Open orders:', len(open_orders_ids)) + try + logger.info(' Open orders:', len(open_orders_ids)) + except TypeError + logger.info('No open orders') return open_orders_ids From dc24d44951fdbef3153cf48ac36fae79792791d7 Mon Sep 17 00:00:00 2001 From: Joshua Greben Date: Tue, 12 Nov 2024 08:19:48 -0800 Subject: [PATCH 5/7] Use logger for errors and print stdout for email log file --- .../folio/encumbrances/fix_encumbrances.py | 96 +++++++++---------- .../encumbrances/fix_encumbrances_run.py | 17 +--- 2 files changed, 49 insertions(+), 64 deletions(-) diff --git a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py index 2169cba5..80e95ef6 100644 --- a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py +++ b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py @@ -48,7 +48,7 @@ def login(tenant, username, password): r = requests.post(okapi_url + '/authn/login', headers=login_headers, json=data) if r.status_code != 201: raise_exception_for_reply(r) - logger.info('Logged in successfully.') + print('Logged in successfully.') okapi_token = r.json()['okapiToken'] return { 'x-okapi-tenant': tenant, @@ -213,24 +213,24 @@ def test_fiscal_year_current(fiscal_year) -> bool: def get_closed_orders_ids() -> list: - logger.info('Retrieving closed order ids...') + print('Retrieving closed order ids...') query = 'workflowStatus=="Closed"' closed_orders_ids = get_order_ids_by_query(query) - try - logger.info(' Closed orders:', len(closed_orders_ids)) - except TypeError - logger.info('No closed orders') + if len(closed_orders_ids): + print(' Closed orders:', len(closed_orders_ids)) + else: + print('No closed orders') return closed_orders_ids def get_open_orders_ids() -> list: - logger.info('Retrieving open order ids...') + print('Retrieving open order ids...') query = 'workflowStatus=="Open"' open_orders_ids = get_order_ids_by_query(query) - try - logger.info(' Open orders:', len(open_orders_ids)) - except TypeError - logger.info('No open orders') + if len(open_orders_ids): + print(' Open orders:', len(open_orders_ids)) + else: + print('No open orders') return open_orders_ids @@ -366,16 +366,16 @@ async def remove_duplicate_encumbrances_in_order(order_id, fiscal_year_id, sem) if len(encumbrance_changes) == 0: sem.release() return 0 - logger.info(f" Removing the following encumbrances for order {order_id}:") + print(f" Removing the following encumbrances for order {order_id}:") for change in encumbrance_changes: - logger.info(f" {change['remove']['id']}") + print(f" {change['remove']['id']}") await remove_encumbrances_and_update_polines(encumbrance_changes) sem.release() return len(encumbrance_changes) async def remove_duplicate_encumbrances(open_and_closed_orders_ids, fiscal_year_id): - logger.info('Removing duplicate encumbrances...') + print('Removing duplicate encumbrances...') futures = [] sem = asyncio.Semaphore(MAX_ACTIVE_THREADS) for idx, order_id in enumerate(open_and_closed_orders_ids): @@ -389,7 +389,7 @@ async def remove_duplicate_encumbrances(open_and_closed_orders_ids, fiscal_year_ nb_removed_encumbrances = await asyncio.gather(*futures) # progress(len(open_and_closed_orders_ids), len(open_and_closed_orders_ids)) - logger.info(f' Removed {sum(nb_removed_encumbrances)} encumbrance(s).') + print(f' Removed {sum(nb_removed_encumbrances)} encumbrance(s).') # --------------------------------------------------- @@ -407,7 +407,7 @@ async def update_encumbrance_fund_id(encumbrance, new_fund_id, poline): encumbrance['fromFundId'] = new_fund_id encumbrance_id = encumbrance['id'] order_id = poline['purchaseOrderId'] - logger.info( + print( f" Fixing fromFundId for po line {poline['id']} ({poline['poLineNumber']}) encumbrance {encumbrance_id}" ) await transaction_summary(order_id, 1) @@ -425,20 +425,20 @@ async def fix_fund_id_with_duplicate_encumbrances(encumbrances, fd_fund_id, poli else: encumbrances_with_bad_fund.append(encumbrance) if len(encumbrances_with_bad_fund) == 0: - logger.warning( + print( f" Warning: there is a remaining duplicate encumbrance for poline {poline['id']} " f"({poline['poLineNumber']})." ) return if len(encumbrances_with_right_fund) != 1: - logger.warning( + print( f" Problem fixing encumbrances for poline {poline['id']} ({poline['poLineNumber']}), " "please fix by hand." ) return replace_by = encumbrances_with_right_fund[0] for encumbrance_to_remove in encumbrances_with_bad_fund: - logger.info( + print( f" Removing encumbrance {encumbrance_to_remove['id']} for po line {poline['id']} " f"({poline['poLineNumber']})" ) @@ -533,12 +533,12 @@ async def process_order_encumbrances_relations(order_id, fiscal_year_id, order_s async def fix_poline_encumbrances_relations( open_orders_ids, fiscal_year_id, fy_is_current ): - logger.info('Fixing poline-encumbrance links...') + print('Fixing poline-encumbrance links...') if len(open_orders_ids) == 0: - logger.info(' Found no open orders.') + print(' Found no open orders.') return if not fy_is_current: - logger.info( + print( ' Fiscal year is not current, the step to fix po line encumbrance relations will be skipped.' ) return @@ -637,9 +637,9 @@ async def fix_order_encumbrances_order_status(order_id, encumbrances): # Eventually we could rely on the post-MODFISTO-328 implementation to change orderStatus directly # (Morning Glory bug fix). try: - # logger.info(f'\n Fixing the following encumbrance(s) for order {order_id} :') + # print(f'\n Fixing the following encumbrance(s) for order {order_id} :') for encumbrance in encumbrances: - logger.info(f" {encumbrance['id']}") + print(f" {encumbrance['id']}") modified_encumbrances = await unrelease_order_encumbrances( order_id, encumbrances ) @@ -669,9 +669,9 @@ async def fix_encumbrance_order_status_for_closed_order( async def fix_encumbrance_order_status_for_closed_orders( closed_orders_ids, fiscal_year_id ): - logger.info('Fixing encumbrance order status for closed orders...') + print('Fixing encumbrance order status for closed orders...') if len(closed_orders_ids) == 0: - logger.info(' Found no closed orders.') + print(' Found no closed orders.') return fix_encumbrance_futures = [] max_active_order_threads = 5 @@ -686,7 +686,7 @@ async def fix_encumbrance_order_status_for_closed_orders( nb_fixed_encumbrances = await asyncio.gather(*fix_encumbrance_futures) # progress(len(closed_orders_ids), len(closed_orders_ids)) - logger.info(f' Fixed order status for {sum(nb_fixed_encumbrances)} encumbrance(s).') + print(f' Fixed order status for {sum(nb_fixed_encumbrances)} encumbrance(s).') # --------------------------------------------------- @@ -694,9 +694,9 @@ async def fix_encumbrance_order_status_for_closed_orders( async def unrelease_encumbrances(order_id, encumbrances): - # logger.info(f'\n Unreleasing the following encumbrance(s) for order {order_id} :') + # print(f'\n Unreleasing the following encumbrance(s) for order {order_id} :') for encumbrance in encumbrances: - logger.info(f" {encumbrance['id']}") + print(f" {encumbrance['id']}") await transaction_summary(order_id, len(encumbrances)) @@ -729,9 +729,9 @@ async def unrelease_encumbrances_with_non_zero_amounts( async def unrelease_open_orders_encumbrances_with_nonzero_amounts( fiscal_year_id, open_orders_ids ): - logger.info('Unreleasing open orders encumbrances with non-zero amounts...') + print('Unreleasing open orders encumbrances with non-zero amounts...') if len(open_orders_ids) == 0: - logger.info(' Found no open orders.') + print(' Found no open orders.') return enc_futures = [] sem = asyncio.Semaphore(MAX_ACTIVE_THREADS) @@ -748,7 +748,7 @@ async def unrelease_open_orders_encumbrances_with_nonzero_amounts( unreleased_encumbrances_amounts = await asyncio.gather(*enc_futures) # progress(len(open_orders_ids), len(open_orders_ids)) - logger.info( + print( f' Unreleased {sum(unreleased_encumbrances_amounts)} open order encumbrance(s) with non-zero amounts.' ) @@ -758,9 +758,9 @@ async def unrelease_open_orders_encumbrances_with_nonzero_amounts( async def release_encumbrances(order_id, encumbrances): - # logger.info\(f'\n Releasing the following encumbrances for order {order_id} :') + # print\(f'\n Releasing the following encumbrances for order {order_id} :') for encumbrance in encumbrances: - logger.info(f" {encumbrance['id']}") + print(f" {encumbrance['id']}") await transaction_summary(order_id, len(encumbrances)) @@ -794,9 +794,9 @@ async def release_encumbrances_with_negative_amounts( async def release_open_orders_encumbrances_with_negative_amounts( fiscal_year_id, open_orders_ids ): - logger.info('Releasing open orders encumbrances with negative amounts...') + print('Releasing open orders encumbrances with negative amounts...') if len(open_orders_ids) == 0: - logger.info(' Found no open orders.') + print(' Found no open orders.') return enc_futures = [] sem = asyncio.Semaphore(MAX_ACTIVE_THREADS) @@ -813,7 +813,7 @@ async def release_open_orders_encumbrances_with_negative_amounts( released_encumbrances_amounts = await asyncio.gather(*enc_futures) # progress(len(open_orders_ids), len(open_orders_ids)) - logger.info( + print( f' Released {sum(released_encumbrances_amounts)} open order encumbrance(s) with negative amounts.' ) @@ -854,9 +854,9 @@ async def release_cancelled_pol_encumbrances(order_id, fiscal_year_id, sem) -> i async def release_cancelled_order_line_encumbrances(fiscal_year_id, open_orders_ids): - logger.info('Releasing cancelled order line encumbrances...') + print('Releasing cancelled order line encumbrances...') if len(open_orders_ids) == 0: - logger.info(' Found no open orders.') + print(' Found no open orders.') return enc_futures = [] sem = asyncio.Semaphore(MAX_ACTIVE_THREADS) @@ -871,7 +871,7 @@ async def release_cancelled_order_line_encumbrances(fiscal_year_id, open_orders_ released_encumbrances_amounts = await asyncio.gather(*enc_futures) # progress(len(open_orders_ids), len(open_orders_ids)) - logger.info( + print( f' Released {sum(released_encumbrances_amounts)} cancelled order line encumbrance(s).' ) @@ -886,7 +886,7 @@ async def update_budgets(encumbered, fund_id, fiscal_year_id, sem) -> int: # Cast into decimal values, so 0 == 0.0 == 0.00 will return true if Decimal(str(budget['encumbered'])) != Decimal(encumbered): - # logger.info(f" Budget \"{budget['name']}\": changing encumbered from {budget['encumbered']} to {encumbered}") + # print(f" Budget \"{budget['name']}\": changing encumbered from {budget['encumbered']} to {encumbered}") budget['encumbered'] = encumbered url = f"{okapi_url}/finance-storage/budgets/{budget['id']}" @@ -899,7 +899,7 @@ async def update_budgets(encumbered, fund_id, fiscal_year_id, sem) -> int: async def recalculate_budget_encumbered(open_and_closed_orders_ids, fiscal_year_id): # Recalculate the encumbered property for all the budgets related to these encumbrances # Take closed orders into account because we might have to set a budget encumbered to 0 - logger.info( + print( f'Recalculating budget encumbered for {len(open_and_closed_orders_ids)} orders ...' ) enc_future = [] @@ -926,7 +926,7 @@ async def recalculate_budget_encumbered(open_and_closed_orders_ids, fiscal_year_ if fund_id in encumbered_for_fund: encumbered_for_fund[fund_id] += Decimal(str(encumbrance['amount'])) - logger.info(' Updating budgets...') + print(' Updating budgets...') update_budget_futures = [] for fund_id, encumbered in encumbered_for_fund.items(): @@ -938,8 +938,8 @@ async def recalculate_budget_encumbered(open_and_closed_orders_ids, fiscal_year_ ) nb_modified = sum(await asyncio.gather(*update_budget_futures)) - logger.info(f' Edited {nb_modified} budget(s).') - logger.info(' Done recalculating budget encumbered.') + print(f' Edited {nb_modified} budget(s).') + print(' Done recalculating budget encumbered.') # --------------------------------------------------- @@ -965,9 +965,9 @@ async def release_order_encumbrances(order_id, fiscal_year_id, sem) -> int: async def release_unreleased_encumbrances_for_closed_orders( closed_orders_ids, fiscal_year_id ): - logger.info('Releasing unreleased encumbrances for closed orders...') + print('Releasing unreleased encumbrances for closed orders...') if len(closed_orders_ids) == 0: - logger.info(' Found no closed orders.') + print(' Found no closed orders.') return nb_released_encumbrance_futures = [] sem = asyncio.Semaphore(MAX_ACTIVE_THREADS) @@ -983,7 +983,7 @@ async def release_unreleased_encumbrances_for_closed_orders( nb_released_encumbrances = await asyncio.gather(*nb_released_encumbrance_futures) # progress(len(closed_orders_ids), len(closed_orders_ids)) - logger.info(f' Released {sum(nb_released_encumbrances)} encumbrance(s).') + print(f' Released {sum(nb_released_encumbrances)} encumbrance(s).') # --------------------------------------------------- diff --git a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances_run.py b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances_run.py index 415fa3ce..1904f515 100644 --- a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances_run.py +++ b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances_run.py @@ -1,6 +1,5 @@ import asyncio import contextlib -import logging import pathlib import libsys_airflow.plugins.folio.encumbrances.fix_encumbrances as fix_encumbrances_script @@ -20,8 +19,7 @@ def fix_encumbrances_run(*args, **kwargs): log_path.parent.mkdir(parents=True, exist_ok=True) with log_path.open("w+", 1) as log: - logging.basicConfig(level=logging.INFO, filename=log) - with contextlib.redirect_stdout(OutputLogger()): + with contextlib.redirect_stdout(log): asyncio.run( fix_encumbrances_script.run_operation( int(choice), fiscal_year_code, tenant, username, password @@ -29,16 +27,3 @@ def fix_encumbrances_run(*args, **kwargs): ) return str(log_path.absolute()) - - -class OutputLogger: - def __init__(self, name="root", level="INFO"): - self.logger = logging.getLogger(name) - self.level = getattr(logging, level) - - def write(self, msg): - if msg and not msg.isspace(): - self.logger.log(self.level, msg) - - def flush(self): - pass From e89433c0f5f23b066cf6dc4898835a5d99e474ff Mon Sep 17 00:00:00 2001 From: Joshua Greben Date: Tue, 12 Nov 2024 09:04:01 -0800 Subject: [PATCH 6/7] Print errors to stfout and also to logger --- .../folio/encumbrances/fix_encumbrances.py | 51 +++++++++++++------ 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py index 80e95ef6..0c7c3b6b 100644 --- a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py +++ b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py @@ -56,8 +56,9 @@ def login(tenant, username, password): 'Content-Type': 'application/json', } except Exception as err: + print('Error during login:', err) logger.error('Error during login:', err) - # raise SystemExit(1) + raise SystemExit(1) async def get_request_without_query(url: str) -> dict: @@ -67,11 +68,13 @@ async def get_request_without_query(url: str) -> dict: if resp.status_code == HTTPStatus.OK: return resp.json() else: + print(f'Error getting record with url {url} : \n{resp.text} ') logger.error(f'Error getting record with url {url} : \n{resp.text} ') - # raise SystemExit(1) + raise SystemExit(1) except Exception as err: + print(f'Error getting record with url {url} : {err=}') logger.error(f'Error getting record with url {url} : {err=}') - # raise SystemExit(1) + raise SystemExit(1) async def get_request(url: str, query: str) -> dict: @@ -85,11 +88,13 @@ async def get_request(url: str, query: str) -> dict: if resp.status_code == HTTPStatus.OK: return resp.json() else: + print(f'Error getting records by {url} ?query= "{query}": \n{resp.text} ') logger.error(f'Error getting records by {url} ?query= "{query}": \n{resp.text} ') - # raise SystemExit(1) + raise SystemExit(1) except Exception as err: + print(f'Error getting records by {url}?query={query}: {err=}') logger.error(f'Error getting records by {url}?query={query}: {err=}') - # raise SystemExit(1) + raise SystemExit(1) async def put_request(url: str, data): @@ -101,12 +106,14 @@ async def put_request(url: str, data): ) if resp.status_code == HTTPStatus.NO_CONTENT: return + print(f'Error updating record {url} "{data}": {resp.text}') logger.error(f'Error updating record {url} "{data}": {resp.text}') - # raise SystemExit(1) + raise SystemExit(1) except Exception as err: + print(f'Error updating record {url} "{data}": {err=}') logger.error(f'Error updating record {url} "{data}": {err=}') - # raise SystemExit(1) + raise SystemExit(1) async def delete_request(url: str): @@ -116,12 +123,14 @@ async def delete_request(url: str): resp = await client.delete(url, headers=headers, timeout=ASYNC_CLIENT_TIMEOUT) if resp.status_code == HTTPStatus.NO_CONTENT: return + print(f'Error deleting record {url}: {resp.text}') logger.error(f'Error deleting record {url}: {resp.text}') - # raise SystemExit(1) + raise SystemExit(1) except Exception as err: + print(f'Error deleting record {url}: {err=}') logger.error(f'Error deleting record {url}: {err=}') - # raise SystemExit(1) + raise SystemExit(1) def get_fiscal_years_by_query(query) -> dict: @@ -134,8 +143,9 @@ def get_fiscal_years_by_query(query) -> dict: raise_exception_for_reply(r) return r.json()['fiscalYears'] except Exception as err: + print(f'Error getting fiscal years with query "{query}": {err}') logger.error(f'Error getting fiscal years with query "{query}": {err}') - # raise SystemExit(1) + raise SystemExit(1) def get_by_chunks(url, query, key) -> list: @@ -173,8 +183,9 @@ def get_order_ids_by_query(query) -> list: for order in orders: ids.append(order['id']) except Exception as err: + print(f'Error getting order ids with query "{query}": {err}') logger.error(f'Error getting order ids with query "{query}": {err}') - # raise SystemExit(1) + raise SystemExit(1) return ids @@ -200,8 +211,9 @@ def get_fiscal_year(fiscal_year_code) -> dict: query = f'code=="{fiscal_year_code}"' fiscal_years = get_fiscal_years_by_query(query) if len(fiscal_years) == 0: + print(f'Could not find fiscal year "{fiscal_year_code}".') logger.error(f'Could not find fiscal year "{fiscal_year_code}".') - # raise SystemExit(1) + raise SystemExit(1) return fiscal_years[0] @@ -249,10 +261,13 @@ async def get_budget_by_fund_id(fund_id, fiscal_year_id) -> dict: query = f'fundId=={fund_id} AND fiscalYearId=={fiscal_year_id}' budgets = await get_budgets_by_query(query) if len(budgets) == 0: + print( + f'Could not find budget for fund "{fund_id}" and fiscal year "{fiscal_year_id}".' + ) logger.error( f'Could not find budget for fund "{fund_id}" and fiscal year "{fiscal_year_id}".' ) - # raise SystemExit(1) + raise SystemExit(1) return budgets[0] @@ -624,10 +639,13 @@ async def fix_order_status_and_release_encumbrances(order_id, encumbrances): await put_request(url, encumbrance) except Exception as err: + print( + f'Error when fixing order status in encumbrances for order {order_id}:', err + ) logger.error( f'Error when fixing order status in encumbrances for order {order_id}:', err ) - # raise SystemExit(1) + raise SystemExit(1) async def fix_order_encumbrances_order_status(order_id, encumbrances): @@ -648,10 +666,13 @@ async def fix_order_encumbrances_order_status(order_id, encumbrances): order_id, modified_encumbrances ) except Exception as err: + print( + f'Error when fixing order status in encumbrances for order {order_id}:', err + ) logger.error( f'Error when fixing order status in encumbrances for order {order_id}:', err ) - # raise SystemExit(1) + raise SystemExit(1) async def fix_encumbrance_order_status_for_closed_order( From 0bfa734681cf92ff820197f6f185bb48b289cb3c Mon Sep 17 00:00:00 2001 From: Joshua Greben Date: Tue, 12 Nov 2024 09:11:53 -0800 Subject: [PATCH 7/7] Only output Logged in successfully message to logger. --- libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py index 0c7c3b6b..cbaa7d34 100644 --- a/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py +++ b/libsys_airflow/plugins/folio/encumbrances/fix_encumbrances.py @@ -48,7 +48,7 @@ def login(tenant, username, password): r = requests.post(okapi_url + '/authn/login', headers=login_headers, json=data) if r.status_code != 201: raise_exception_for_reply(r) - print('Logged in successfully.') + logger.info('Logged in successfully.') okapi_token = r.json()['okapiToken'] return { 'x-okapi-tenant': tenant,