From ad3563dae2f51815c0cc15ec5488ccdffd7a0536 Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Sun, 2 Jan 2022 20:17:50 +0100 Subject: [PATCH 01/26] updated Kraken mapping --- src/core.py | 69 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/src/core.py b/src/core.py index 67326ec4..e1d11dd7 100644 --- a/src/core.py +++ b/src/core.py @@ -202,32 +202,41 @@ class Fiat(Enum): # Converts clean fiat / clean crypto pairs to "dirty" API asset pairs # (e.g. ETHEUR -> XETHZEUR) # Analyzed using asset pairs API data: -# https://api.kraken.com/0/public/AssetPairs (retrieved at 2021-02-21) +# https://api.kraken.com/0/public/AssetPairs (retrieved at 2022-01-02) kraken_asset_map = { # Fiat: - "ZUSD": "USD", - "ZEUR": "EUR", - "ZCAD": "CAD", - "ZJPY": "JPY", - "ZGBP": "GBP", "CHF": "CHF", "ZAUD": "AUD", + "ZCAD": "CAD", + "ZEUR": "EUR", + "ZGBP": "GBP", + "ZJPY": "JPY", + "ZUSD": "USD", # Crypto: - "XXBT": "XBT", + "XETC": "ETC", "XETH": "ETH", "XLTC": "LTC", + "XMLN": "MLN", + "XREP": "REP", + "XXBT": "XBT", "XXDG": "XDG", + "XXLM": "XLM", + "XXMR": "XMR", "XXRP": "XRP", + "XZEC": "ZEC", } # Only these asset pairs violate the rule # "clean name" + "clean name" = "asset pair" kraken_pair_map = { "USDTUSD": "USDTZUSD", - "XETCETH": "XETCXETH", - "XETCXBT": "XETCXXBT", - "XETCEUR": "XETCZEUR", - "XETCUSD": "XETCZUSD", + "ETCETH": "XETCXETH", + "ETCXBT": "XETCXXBT", + "ETCEUR": "XETCZEUR", + "ETCUSD": "XETCZUSD", + "ETH2ETH": "ETH2.SETH", + "ETH2EUR": "XETHZEUR", + "ETH2USD": "XETHZUSD", "ETHXBT": "XETHXXBT", "ETHCAD": "XETHZCAD", "ETHEUR": "XETHZEUR", @@ -238,36 +247,36 @@ class Fiat(Enum): "LTCEUR": "XLTCZEUR", "LTCJPY": "XLTCZJPY", "LTCUSD": "XLTCZUSD", - "XMLNETH": "XMLNXETH", - "XMLNXBT": "XMLNXXBT", - "XMLNEUR": "XMLNZEUR", - "XMLNUSD": "XMLNZUSD", - "XREPETH": "XREPXETH", - "XREPXBT": "XREPXXBT", - "XREPEUR": "XREPZEUR", - "XREPUSD": "XREPZUSD", + "MLNETH": "XMLNXETH", + "MLNXBT": "XMLNXXBT", + "MLNEUR": "XMLNZEUR", + "MLNUSD": "XMLNZUSD", + "REPETH": "XREPXETH", + "REPXBT": "XREPXXBT", + "REPEUR": "XREPZEUR", + "REPUSD": "XREPZUSD", "XBTCAD": "XXBTZCAD", "XBTEUR": "XXBTZEUR", "XBTGBP": "XXBTZGBP", "XBTJPY": "XXBTZJPY", "XBTUSD": "XXBTZUSD", "XDGXBT": "XXDGXXBT", - "XXLMXBT": "XXLMXXBT", - "XXLMAUD": "XXLMZAUD", - "XXLMEUR": "XXLMZEUR", - "XXLMGBP": "XXLMZGBP", - "XXLMUSD": "XXLMZUSD", - "XXMRXBT": "XXMRXXBT", - "XXMREUR": "XXMRZEUR", - "XXMRUSD": "XXMRZUSD", + "XLMXBT": "XXLMXXBT", + "XLMAUD": "XXLMZAUD", + "XLMEUR": "XXLMZEUR", + "XLMGBP": "XXLMZGBP", + "XLMUSD": "XXLMZUSD", + "XMRXBT": "XXMRXXBT", + "XMREUR": "XXMRZEUR", + "XMRUSD": "XXMRZUSD", "XRPXBT": "XXRPXXBT", "XRPCAD": "XXRPZCAD", "XRPEUR": "XXRPZEUR", "XRPJPY": "XXRPZJPY", "XRPUSD": "XXRPZUSD", - "XZECXBT": "XZECXXBT", - "XZECEUR": "XZECZEUR", - "XZECUSD": "XZECZUSD", + "ZECXBT": "XZECXXBT", + "ZECEUR": "XZECZEUR", + "ZECUSD": "XZECZUSD", "EURUSD": "ZEURZUSD", "GBPUSD": "ZGBPZUSD", "USDCAD": "ZUSDZCAD", From 75e027d372093f902af5ac98aef1feae767f146d Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Sun, 2 Jan 2022 21:42:04 +0100 Subject: [PATCH 02/26] different way of handling Kraken exports --- src/book.py | 81 ++++++++++++++++++++++++++--------------------------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/src/book.py b/src/book.py index 51601ed4..f0dd9643 100644 --- a/src/book.py +++ b/src/book.py @@ -452,30 +452,15 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: operation_mapping = { "spend": "Sell", # Sell ordered via 'Buy Crypto' button "receive": "Buy", # Buy ordered via 'Buy Crypto' button - "transfer": "Airdrop", "reward": "StakingInterest", + "staking": "StakingInterest", "deposit": "Deposit", "withdrawal": "Withdrawal", } - # Need to track state of "duplicate entries" - # for deposits / withdrawals; - # the second deposit and the first withdrawal entry - # need to be skipped. - # dup_state["deposit"] == 0: - # Deposit is broadcast to blockchain - # > Taxable event (is in public trade history) - # dup_state["deposit"] == 1: - # Deposit is credited to Kraken account - # > Skipped - # dup_state["withdrawal"] == 0: - # Withdrawal is requested in Kraken account - # > Skipped - # dup_state["withdrawal"] == 1: - # Withdrawal is broadcast to blockchain - # > Taxable event (is in public trade history) - dup_state = {"deposit": 0, "withdrawal": 0} - dup_skip = {"deposit": 1, "withdrawal": 0} + # Need to track state of duplicate entries + # for deposits / withdrawals based on refid + refids = [] with open(file_path, encoding="utf8") as f: reader = csv.reader(f) @@ -522,35 +507,57 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: row = reader.line_num - # Skip "duplicate entries" for deposits / withdrawals - if _type in dup_state.keys(): - skip = dup_state[_type] == dup_skip[_type] - dup_state[_type] = (dup_state[_type] + 1) % 2 - if skip: + # Skip duplicate entries for deposits / withdrawals and + # additional deposit / withdrawals lines for + # staking / unstaking / staking reward actions + if _type in ["deposit", "withdrawal"]: + if refid not in refids: + refids.append(refid) continue # Parse data. utc_time = datetime.datetime.strptime(_utc_time, "%Y-%m-%d %H:%M:%S") utc_time = utc_time.replace(tzinfo=datetime.timezone.utc) change = misc.force_decimal(_amount) + # remove the appended .S for staked assets + _asset = _asset.replace(".S", "") coin = kraken_asset_map.get(_asset, _asset) fee = misc.force_decimal(_fee) operation = operation_mapping.get(_type) if operation is None: if _type == "trade": operation = "Sell" if change < 0 else "Buy" - elif _type in ["margin trade", "rollover", "settled"]: - log.error( - f"{file_path}: {row}: Margin trading is " - "currently not supported. " - "Please create an Issue or PR." + elif _type in ["margin trade", "rollover", "settled", "margin"]: + log.warning( + f"{file_path}: {row}: Margin trading is currently not " + "supported. Please create an Issue or PR." ) - raise RuntimeError + continue + elif _type == "transfer": + if num_columns == 9: + # for backwards compatibility assume Airdrop for staking + log.warning( + f"{file_path}: {row}: Staking is not supported for old" + " Kraken ledger formats. Please create an Issue or PR." + ) + operation = "Airdrop" + elif subtype == "stakingfromspot": + operation = "Staking" + elif subtype == "stakingtospot": + operation = "StakingEnd" + elif subtype in ["spottostaking", "spotfromstaking"]: + # duplicate entries for staking actions + continue + else: + log.error( + f"{file_path}: {row}: Order subtype '{subtype}' is " + "currently not supported. Please create an Issue or PR." + ) + raise RuntimeError else: log.error( - f"{file_path}: {row}: Other order type '{_type}' " - "is currently not supported. " - "Please create an Issue or PR." + f"{file_path}: {row}: Other order type '{_type}' is " + "currently not supported. Please create an Issue or PR." ) raise RuntimeError change = abs(change) @@ -569,14 +576,6 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: "Fee", utc_time, platform, fee, coin, row, file_path ) - assert dup_state["deposit"] == 0, ( - "Orphaned deposit. (Must always come in pairs). " "Is your file corrupted?" - ) - assert dup_state["withdrawal"] == 0, ( - "Orphaned withdrawal. (Must always come in pairs). " - "Is your file corrupted?" - ) - def _read_kraken_ledgers_old(self, file_path: Path) -> None: self._read_kraken_ledgers(file_path) From 7cef9acc067419f31d39edfb621adc9f4be86a25 Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Sun, 2 Jan 2022 21:45:33 +0100 Subject: [PATCH 03/26] removed trailing whitespace --- src/book.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/book.py b/src/book.py index f0dd9643..b000aa36 100644 --- a/src/book.py +++ b/src/book.py @@ -508,7 +508,7 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: row = reader.line_num # Skip duplicate entries for deposits / withdrawals and - # additional deposit / withdrawals lines for + # additional deposit / withdrawals lines for # staking / unstaking / staking reward actions if _type in ["deposit", "withdrawal"]: if refid not in refids: From d49bd63a8e31da4f6c6b2ff67365b72e20986963 Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Mon, 3 Jan 2022 12:23:06 +0100 Subject: [PATCH 04/26] try inverse pair if Kraken pair is invalid --- src/price_data.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/price_data.py b/src/price_data.py index 9dcabef5..200a0151 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -336,8 +336,7 @@ def _get_price_kraken( """ target_timestamp = misc.to_ms_timestamp(utc_time) root_url = "https://api.kraken.com/0/public/Trades" - pair = base_asset + quote_asset - pair = kraken_pair_map.get(pair, pair) + inverse = False minutes_offset = 0 while minutes_offset < 120: @@ -346,10 +345,13 @@ def _get_price_kraken( since = misc.to_ns_timestamp( utc_time - datetime.timedelta(minutes=minutes_offset) ) - url = f"{root_url}?{pair=:}&{since=:}" num_retries = 10 while num_retries: + pair = base_asset + quote_asset + pair = kraken_pair_map.get(pair, pair) + url = f"{root_url}?{pair=:}&{since=:}" + log.debug( f"Querying trades for {pair} at {utc_time} " f"(offset={minutes_offset}m): Calling %s", @@ -361,6 +363,18 @@ def _get_price_kraken( if not data["error"]: break + elif data["error"] == ['EGeneral:Invalid arguments']: + # cancel if inverse pair has been tried + if inverse: + num_retries = 0 + continue + # try inverse pair + else: + log.debug("Invalid arguments error, trying inverse coin pair") + inverse = not inverse + temp_asset = quote_asset + quote_asset = base_asset + base_asset = temp_asset else: num_retries -= 1 sleep_duration = 2 ** (10 - num_retries) @@ -408,6 +422,8 @@ def _get_price_kraken( ) price = misc.force_decimal(data[closest_match_index][0]) + if inverse: + price = misc.reciprocal(price) return price log.warning( From fbf0609d7a6d5bf8164effbe1c4464fe1e879715 Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Mon, 3 Jan 2022 13:05:20 +0100 Subject: [PATCH 05/26] reduce number of margin trading warnings --- src/book.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/book.py b/src/book.py index b000aa36..b2dd7229 100644 --- a/src/book.py +++ b/src/book.py @@ -461,6 +461,7 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: # Need to track state of duplicate entries # for deposits / withdrawals based on refid refids = [] + margin_warnings = 0 with open(file_path, encoding="utf8") as f: reader = csv.reader(f) @@ -528,10 +529,7 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: if _type == "trade": operation = "Sell" if change < 0 else "Buy" elif _type in ["margin trade", "rollover", "settled", "margin"]: - log.warning( - f"{file_path}: {row}: Margin trading is currently not " - "supported. Please create an Issue or PR." - ) + margin_warnings += 1 continue elif _type == "transfer": if num_columns == 9: @@ -576,6 +574,12 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: "Fee", utc_time, platform, fee, coin, row, file_path ) + if margin_warnings: + log.warning( + f"{file_path}: {margin_warnings} margin entries. Margin trading is " + "currently not supported. Please create an Issue or PR." + ) + def _read_kraken_ledgers_old(self, file_path: Path) -> None: self._read_kraken_ledgers(file_path) From 9f377f3a75c1c100273572cacdd59ac74263e9f0 Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Mon, 3 Jan 2022 13:50:40 +0100 Subject: [PATCH 06/26] allow future timestamp for virtual sells --- src/price_data.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/price_data.py b/src/price_data.py index 200a0151..2611e0c7 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -405,8 +405,12 @@ def _get_price_kraken( if closest_match_index == -1: continue - # The desired timestamp is in the future - if closest_match_index == len(data_timestamps_ms) - 1: + now_timestamp = misc.to_ms_timestamp(datetime.datetime.now().astimezone()) + # The desired timestamp is in the future. Ignore if the target timestamp + # is very close to the current timestamp (for virtual sells) + if closest_match_index == len(data_timestamps_ms) - 1 and \ + (now_timestamp > target_timestamp + 3600 * 1000 + or not config.CALCULATE_VIRTUAL_SELL): if minutes_step == 1: # Cannot remove interval any further; give up From c71d9e157a427c875ea3b2c34d253c311a77334f Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Tue, 4 Jan 2022 21:13:14 +0100 Subject: [PATCH 07/26] address review comments - WIP --- src/book.py | 2 +- src/price_data.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/book.py b/src/book.py index b2dd7229..7ee349fa 100644 --- a/src/book.py +++ b/src/book.py @@ -521,7 +521,7 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: utc_time = utc_time.replace(tzinfo=datetime.timezone.utc) change = misc.force_decimal(_amount) # remove the appended .S for staked assets - _asset = _asset.replace(".S", "") + _asset = _asset.removesuffix(".S") coin = kraken_asset_map.get(_asset, _asset) fee = misc.force_decimal(_fee) operation = operation_mapping.get(_type) diff --git a/src/price_data.py b/src/price_data.py index 2611e0c7..00b31f94 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -372,9 +372,7 @@ def _get_price_kraken( else: log.debug("Invalid arguments error, trying inverse coin pair") inverse = not inverse - temp_asset = quote_asset - quote_asset = base_asset - base_asset = temp_asset + base_asset, quote_asset = quote_asset, base_asset else: num_retries -= 1 sleep_duration = 2 ** (10 - num_retries) @@ -407,7 +405,7 @@ def _get_price_kraken( now_timestamp = misc.to_ms_timestamp(datetime.datetime.now().astimezone()) # The desired timestamp is in the future. Ignore if the target timestamp - # is very close to the current timestamp (for virtual sells) + # is within one hour to the current timestamp (for virtual sells) if closest_match_index == len(data_timestamps_ms) - 1 and \ (now_timestamp > target_timestamp + 3600 * 1000 or not config.CALCULATE_VIRTUAL_SELL): From e9d809dd9d1d80ce0309d5ab4c6c6e8ade0d62da Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Wed, 5 Jan 2022 20:06:43 +0100 Subject: [PATCH 08/26] reworked Kraken API warnings/errors --- src/price_data.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/price_data.py b/src/price_data.py index 00b31f94..992eb93b 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -364,31 +364,37 @@ def _get_price_kraken( if not data["error"]: break elif data["error"] == ['EGeneral:Invalid arguments']: - # cancel if inverse pair has been tried - if inverse: - num_retries = 0 - continue # try inverse pair - else: - log.debug("Invalid arguments error, trying inverse coin pair") + if not inverse: + log.warning( + f"Invalid arguments error for {pair} at {utc_time} " + f"(offset={minutes_offset}m): " + f"Trying inverse coin pair ..." + ) inverse = not inverse base_asset, quote_asset = quote_asset, base_asset + # cancel if inverse pair has been tried + else: + log.error( + f"Could not retrieve trades for {pair} or inverse pair at " + f"{utc_time} (offset={minutes_offset}m): " + "Invalid arguments error. Please create an Issue or PR." + ) + raise RuntimeError else: num_retries -= 1 sleep_duration = 2 ** (10 - num_retries) log.warning( - f"Querying trades for {pair} at {utc_time} " - f"(offset={minutes_offset}m): " - f"Could not retrieve trades: {data['error']}. " + f"Could not retrieve trades for {pair} at {utc_time} " + f"(offset={minutes_offset}m): {data['error']}. " f"Retry in {sleep_duration} s ..." ) time.sleep(sleep_duration) continue else: log.error( - f"Querying trades for {pair} at {utc_time} " - f"(offset={minutes_offset}m): " - f"Could not retrieve trades: {data['error']}" + f"Could not retrieve trades for {pair} at {utc_time} " + f"(offset={minutes_offset}m): {data['error']}. " ) raise RuntimeError("Kraken response keeps having error flags.") @@ -429,8 +435,7 @@ def _get_price_kraken( return price log.warning( - f"Querying trades for {pair} at {utc_time}: " - f"Failed to find matching exchange rate. " + f"Failed to find matching exchange rate for {pair} at {utc_time}: " "Please create an Issue or PR." ) return decimal.Decimal() From 1b785441ecf7cb1357e159fea509fe210eb83bb1 Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Thu, 6 Jan 2022 15:43:20 +0100 Subject: [PATCH 09/26] throw error for margin trades --- src/book.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/book.py b/src/book.py index 7ee349fa..7df647ab 100644 --- a/src/book.py +++ b/src/book.py @@ -461,7 +461,6 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: # Need to track state of duplicate entries # for deposits / withdrawals based on refid refids = [] - margin_warnings = 0 with open(file_path, encoding="utf8") as f: reader = csv.reader(f) @@ -529,8 +528,11 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: if _type == "trade": operation = "Sell" if change < 0 else "Buy" elif _type in ["margin trade", "rollover", "settled", "margin"]: - margin_warnings += 1 - continue + log.error( + f"{file_path}: {row}: Margin trading is currently not " + "supported. Please create an Issue or PR." + ) + raise RuntimeError elif _type == "transfer": if num_columns == 9: # for backwards compatibility assume Airdrop for staking @@ -574,12 +576,6 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: "Fee", utc_time, platform, fee, coin, row, file_path ) - if margin_warnings: - log.warning( - f"{file_path}: {margin_warnings} margin entries. Margin trading is " - "currently not supported. Please create an Issue or PR." - ) - def _read_kraken_ledgers_old(self, file_path: Path) -> None: self._read_kraken_ledgers(file_path) From 883c3e5c452e673915972d2e2de290dd41a349ec Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Sat, 15 Jan 2022 17:02:54 +0100 Subject: [PATCH 10/26] updated handling of Kraken deposits/withdrawals --- src/book.py | 118 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 95 insertions(+), 23 deletions(-) diff --git a/src/book.py b/src/book.py index 7df647ab..1f907829 100644 --- a/src/book.py +++ b/src/book.py @@ -458,9 +458,9 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: "withdrawal": "Withdrawal", } - # Need to track state of duplicate entries - # for deposits / withdrawals based on refid - refids = [] + # Need to track state of duplicate deposit/withdrawal entries + # All deposits/withdrawals are held back until they occur a second time + held_operations = [] with open(file_path, encoding="utf8") as f: reader = csv.reader(f) @@ -500,21 +500,14 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: balance, ) = columns else: - raise RuntimeError( - "Unknown Kraken ledgers format: " + log.error( + "{file_path}: Unknown Kraken ledgers format: " "Number of rows do not match known versions." ) + raise RuntimeError row = reader.line_num - # Skip duplicate entries for deposits / withdrawals and - # additional deposit / withdrawals lines for - # staking / unstaking / staking reward actions - if _type in ["deposit", "withdrawal"]: - if refid not in refids: - refids.append(refid) - continue - # Parse data. utc_time = datetime.datetime.strptime(_utc_time, "%Y-%m-%d %H:%M:%S") utc_time = utc_time.replace(tzinfo=datetime.timezone.utc) @@ -529,7 +522,7 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: operation = "Sell" if change < 0 else "Buy" elif _type in ["margin trade", "rollover", "settled", "margin"]: log.error( - f"{file_path}: {row}: Margin trading is currently not " + f"{file_path} row {row}: Margin trading is currently not " "supported. Please create an Issue or PR." ) raise RuntimeError @@ -537,8 +530,9 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: if num_columns == 9: # for backwards compatibility assume Airdrop for staking log.warning( - f"{file_path}: {row}: Staking is not supported for old" - " Kraken ledger formats. Please create an Issue or PR." + f"{file_path} row {row}: Staking is not supported for" + "old Kraken ledger formats. " + "Please create an Issue or PR." ) operation = "Airdrop" elif subtype == "stakingfromspot": @@ -550,13 +544,13 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: continue else: log.error( - f"{file_path}: {row}: Order subtype '{subtype}' is " + f"{file_path} row {row}: Order subtype '{subtype}' is " "currently not supported. Please create an Issue or PR." ) raise RuntimeError else: log.error( - f"{file_path}: {row}: Other order type '{_type}' is " + f"{file_path} row {row}: Other order type '{_type}' is " "currently not supported. Please create an Issue or PR." ) raise RuntimeError @@ -567,14 +561,92 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: assert coin assert change - self.append_operation( - operation, utc_time, platform, change, coin, row, file_path - ) + # append all operations to main list per default + # will be overwritten for deposits / withdrawals + append_operation = True + + # Skip duplicate entries for deposits / withdrawals and additional + # deposit / withdrawal lines for staking / unstaking / staking reward + # actions. + # The second deposit and the first withdrawal need to be considered, + # since these are the points in time where the user actually has the + # assets at their disposal. The first deposit and second withdrawal are + # in the public trade history and are skipped. + # For staking / unstaking / staking reward actions, deposits / + # withdrawals only occur once and will be ignored. + if operation in ["Deposit", "Withdrawal"]: + # search for refid in refids list + refid_idxs = [ + idx for idx, op in enumerate(held_operations) \ + if op["refid"] == refid + ] + # refid should not exist more than once in list + if len(refid_idxs) > 1: + log.error( + f"{file_path} row {row}: More than two entries with refid " + f"{refid} should not exist ({operation}). " + "Please create an Issue or PR." + ) + raise RuntimeError + # if refid already exists, assert that data of operations agree + if refid_idxs: + idx = refid_idxs[0] + try: + assert operation == held_operations[idx]["operation"] + assert change == held_operations[idx]["change"] + assert coin == held_operations[idx]["coin"] + except AssertionError: + log.error( + f"{file_path} row {row}: Parameters for refid {refid} " + f"({operation}) do not agree. " + "Please create an Issue or PR." + ) + raise RuntimeError + # add all entries to refid list and held operations list + held_operations.append({ + "refid": refid, + "operation": operation, + "utc_time": utc_time, + "platform": platform, + "change": change, + "coin": coin, + "row": row, + "file_path": file_path, + "fee": fee, + }) + + if operation == "Deposit": + # append only the second deposit to the operations list + if refid_idxs: + append_operation = True + else: + append_operation = False + elif operation == "Withdrawal": + # Append the first withdrawal to the operations list as soon + # as the second withdrawal occurs. Therefore, for the second + # withdrawal, overwrite the variables with the data of the first + # withdrawal and append this to the operations list. + if refid_idxs: + append_operation = True + operation = held_operations[idx]["operation"] + utc_time = held_operations[idx]["utc_time"] + platform = held_operations[idx]["platform"] + change = held_operations[idx]["change"] + coin = held_operations[idx]["coin"] + row = held_operations[idx]["row"] + file_path = held_operations[idx]["file_path"] + fee = held_operations[idx]["fee"] + else: + append_operation = False - if fee != 0: + if append_operation: self.append_operation( - "Fee", utc_time, platform, fee, coin, row, file_path + operation, utc_time, platform, change, coin, row, file_path ) + if fee != 0: + self.append_operation( + "Fee", utc_time, platform, fee, coin, row, file_path + ) def _read_kraken_ledgers_old(self, file_path: Path) -> None: From 4722b632932f9d18c418bd368008b1aede687e30 Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Sun, 16 Jan 2022 01:27:21 +0100 Subject: [PATCH 11/26] updated Kraken API handling of latest trade prices --- src/book.py | 4 ++-- src/price_data.py | 40 +++++++++++++++++++++++++++++----------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/book.py b/src/book.py index 1f907829..af8ebc70 100644 --- a/src/book.py +++ b/src/book.py @@ -572,12 +572,12 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: # since these are the points in time where the user actually has the # assets at their disposal. The first deposit and second withdrawal are # in the public trade history and are skipped. - # For staking / unstaking / staking reward actions, deposits / + # For staking / unstaking / staking reward actions, deposits / # withdrawals only occur once and will be ignored. if operation in ["Deposit", "Withdrawal"]: # search for refid in refids list refid_idxs = [ - idx for idx, op in enumerate(held_operations) \ + idx for idx, op in enumerate(held_operations) if op["refid"] == refid ] # refid should not exist more than once in list diff --git a/src/price_data.py b/src/price_data.py index 992eb93b..e4d853e6 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -315,10 +315,12 @@ def _get_price_kraken( For this we fetch one chunk of the trade history, starting `minutes_step`minutes before this timestamp. We then walk through the history until the closest timestamp match is - found. Otherwise, we start another 10 minutes earlier and try again. + found. Otherwise (if all received price data points are newer than the desired + timestamp), we start another 10 minutes earlier and try again. (Exiting with a warning and zero price after hitting the arbitrarily chosen offset limit of 120 minutes). If the initial offset is already - too large, recursively retry by reducing the offset step, + too large (i.e. all received price data points are older than the desired + timestamp), recursively retry by reducing the offset step, down to 1 minute. Documentation: https://www.kraken.com/features/api @@ -406,18 +408,34 @@ def _get_price_kraken( ) # The desired timestamp is in the past; increase the offset + # desired timestamp is smaller than all timestamps of the received data if closest_match_index == -1: continue - now_timestamp = misc.to_ms_timestamp(datetime.datetime.now().astimezone()) - # The desired timestamp is in the future. Ignore if the target timestamp - # is within one hour to the current timestamp (for virtual sells) - if closest_match_index == len(data_timestamps_ms) - 1 and \ - (now_timestamp > target_timestamp + 3600 * 1000 - or not config.CALCULATE_VIRTUAL_SELL): - - if minutes_step == 1: - # Cannot remove interval any further; give up + # The desired timestamp is in the future + # desired timestamp is larger than all timestamps of the received data + if closest_match_index == len(data_timestamps_ms) - 1: + if len(data_timestamps_ms) < 100: + # The API returns the last 1000 trades. If less than 100 trades are + # received, it can be assumed that we've received the last trade. + price_timestamp = data_timestamps_ms[closest_match_index] + log.debug( + "Accepting price from " + f"{datetime.datetime.fromtimestamp(price_timestamp/1000.0)} " + f"as latest price for {pair} at {utc_time}" + ) + # This should normally only happen for virtual sells, therefore + # raise a warning if the target timestamp is older than one hour + now_timestamp = misc.to_ms_timestamp( + datetime.datetime.now().astimezone() + ) + if target_timestamp < now_timestamp - 3600 * 1000: + log.warning( + f"Timestamp for {pair} at {utc_time} is older than one " + "hour, still accepted latest received trading price" + ) + elif minutes_step == 1: + # Cannot reduce interval any further; give up break else: # We missed the desired timestamp because our initial step From 58701345a75d31c0026e7baa48e080f778859605 Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Sun, 23 Jan 2022 15:01:18 +0100 Subject: [PATCH 12/26] updated type annotation --- src/book.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/book.py b/src/book.py index af8ebc70..6e9b48fa 100644 --- a/src/book.py +++ b/src/book.py @@ -20,7 +20,7 @@ import logging import re from pathlib import Path -from typing import Optional +from typing import Any, Optional import config import misc @@ -460,7 +460,7 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: # Need to track state of duplicate deposit/withdrawal entries # All deposits/withdrawals are held back until they occur a second time - held_operations = [] + held_operations: list[dict[str, Any]] = [] with open(file_path, encoding="utf8") as f: reader = csv.reader(f) From c03b90f888ef5e2947dd0d4169361696e1b6df49 Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Sun, 23 Jan 2022 15:04:56 +0100 Subject: [PATCH 13/26] added Optional[str] to str conversion --- src/book.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/book.py b/src/book.py index 6e9b48fa..863ca792 100644 --- a/src/book.py +++ b/src/book.py @@ -628,7 +628,8 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: # withdrawal and append this to the operations list. if refid_idxs: append_operation = True - operation = held_operations[idx]["operation"] + # required for type annotation: convert Optional[str] to str + operation = str(held_operations[idx]["operation"]) utc_time = held_operations[idx]["utc_time"] platform = held_operations[idx]["platform"] change = held_operations[idx]["change"] From 31cba9a06944b158d5f0402b4643437da1c46d3d Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Wed, 26 Jan 2022 22:04:58 +0100 Subject: [PATCH 14/26] stake rewarded coins --- src/book.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/book.py b/src/book.py index 863ca792..8bd065fe 100644 --- a/src/book.py +++ b/src/book.py @@ -648,6 +648,12 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: self.append_operation( "Fee", utc_time, platform, fee, coin, row, file_path ) + if operation == "StakingInterest": + # the rewarded coins are added to the staked portfolio + # TODO: directly stake the rewarded coins, not just any coin + self.append_operation( + "Staking", utc_time, platform, change, coin, row, file_path + ) def _read_kraken_ledgers_old(self, file_path: Path) -> None: From 1870cd1e47b51ba3a27d4212f3114994abcb1d33 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 16:02:32 +0100 Subject: [PATCH 15/26] UPDATE Specify error message when refid parameters doe not match --- src/book.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/book.py b/src/book.py index 8bd065fe..4ff2dcaf 100644 --- a/src/book.py +++ b/src/book.py @@ -592,13 +592,15 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: if refid_idxs: idx = refid_idxs[0] try: - assert operation == held_operations[idx]["operation"] - assert change == held_operations[idx]["change"] - assert coin == held_operations[idx]["coin"] - except AssertionError: + assert ( + operation == held_operations[idx]["operation"] + ), "operation" + assert change == held_operations[idx]["change"], "change" + assert coin == held_operations[idx]["coin"], "coin" + except AssertionError as e: log.error( f"{file_path} row {row}: Parameters for refid {refid} " - f"({operation}) do not agree. " + f"({operation}) do not agree: {e} " "Please create an Issue or PR." ) raise RuntimeError From ca11a5d7099c9b7e16c2951819d65b757fc74d76 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 16:02:55 +0100 Subject: [PATCH 16/26] UPDATE autoformat --- src/book.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/book.py b/src/book.py index 4ff2dcaf..aeff1587 100644 --- a/src/book.py +++ b/src/book.py @@ -577,7 +577,8 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: if operation in ["Deposit", "Withdrawal"]: # search for refid in refids list refid_idxs = [ - idx for idx, op in enumerate(held_operations) + idx + for idx, op in enumerate(held_operations) if op["refid"] == refid ] # refid should not exist more than once in list @@ -605,17 +606,19 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: ) raise RuntimeError # add all entries to refid list and held operations list - held_operations.append({ - "refid": refid, - "operation": operation, - "utc_time": utc_time, - "platform": platform, - "change": change, - "coin": coin, - "row": row, - "file_path": file_path, - "fee": fee, - }) + held_operations.append( + { + "refid": refid, + "operation": operation, + "utc_time": utc_time, + "platform": platform, + "change": change, + "coin": coin, + "row": row, + "file_path": file_path, + "fee": fee, + } + ) if operation == "Deposit": # append only the second deposit to the operations list From adb45e4ff4f27df59e17dc5bdf81e91234730e48 Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Sun, 6 Feb 2022 20:16:52 +0100 Subject: [PATCH 17/26] UPDATE deposit logic, ADD comment for staking rewards --- src/book.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/book.py b/src/book.py index aeff1587..694dba6f 100644 --- a/src/book.py +++ b/src/book.py @@ -622,10 +622,7 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: if operation == "Deposit": # append only the second deposit to the operations list - if refid_idxs: - append_operation = True - else: - append_operation = False + append_operation = len(refid_idxs) == 1 elif operation == "Withdrawal": # Append the first withdrawal to the operations list as soon # as the second withdrawal occurs. Therefore, for the second @@ -654,8 +651,10 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: "Fee", utc_time, platform, fee, coin, row, file_path ) if operation == "StakingInterest": - # the rewarded coins are added to the staked portfolio - # TODO: directly stake the rewarded coins, not just any coin + # For Kraken, the rewarded coins are added to the staked + # portfolio. TODO (for MULTI_DEPOT only): Directly add the + # rewarded coins to the staking depot (not like here with the + # detour of adding it to spot and then staking the same amount) self.append_operation( "Staking", utc_time, platform, change, coin, row, file_path ) From e250176524d1461ada287401fcc9838f033edf91 Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Sat, 19 Feb 2022 12:39:28 +0100 Subject: [PATCH 18/26] REFACTOR store invalid API Kraken pairs in list and don't try them again --- src/price_data.py | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/price_data.py b/src/price_data.py index e4d853e6..b5b330a4 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -41,6 +41,9 @@ class PriceData: + # list of Kraken pairs that returned invalid arguments error + kraken_invalid_pairs: list[str] = [] + def get_db_path(self, platform: str) -> Path: return Path(config.DATA_PATH, f"{platform}.db") @@ -352,6 +355,21 @@ def _get_price_kraken( while num_retries: pair = base_asset + quote_asset pair = kraken_pair_map.get(pair, pair) + + # if the pair is invalid, invert it + if pair in self.kraken_invalid_pairs: + inverse = not inverse + base_asset, quote_asset = quote_asset, base_asset + pair = base_asset + quote_asset + pair = kraken_pair_map.get(pair, pair) + # if inverted pair is also invalid, throw error + if pair in self.kraken_invalid_pairs: + log.error( + f"Could not retrieve trades for {pair} or inverse pair, " + "invalid arguments error. Please create an Issue or PR." + ) + raise RuntimeError + url = f"{root_url}?{pair=:}&{since=:}" log.debug( @@ -366,23 +384,14 @@ def _get_price_kraken( if not data["error"]: break elif data["error"] == ['EGeneral:Invalid arguments']: - # try inverse pair - if not inverse: - log.warning( - f"Invalid arguments error for {pair} at {utc_time} " - f"(offset={minutes_offset}m): " - f"Trying inverse coin pair ..." - ) - inverse = not inverse - base_asset, quote_asset = quote_asset, base_asset - # cancel if inverse pair has been tried - else: - log.error( - f"Could not retrieve trades for {pair} or inverse pair at " - f"{utc_time} (offset={minutes_offset}m): " - "Invalid arguments error. Please create an Issue or PR." - ) - raise RuntimeError + # add pair to invalid pairs list + # leads to inversion of pair next time + log.warning( + f"Invalid arguments error for {pair} at {utc_time} " + f"(offset={minutes_offset}m): " + f"Blocking pair and trying inverse coin pair ..." + ) + self.kraken_invalid_pairs.append(pair) else: num_retries -= 1 sleep_duration = 2 ** (10 - num_retries) From f002f02b8ed3b3eecd0e7d2509a0d977048f584a Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Sat, 19 Feb 2022 15:25:06 +0100 Subject: [PATCH 19/26] REFACTOR handling of Kraken deposits/withdrawals --- src/book.py | 129 ++++++++++++++++++++++++++++------------------------ 1 file changed, 69 insertions(+), 60 deletions(-) diff --git a/src/book.py b/src/book.py index 694dba6f..8a795854 100644 --- a/src/book.py +++ b/src/book.py @@ -19,6 +19,7 @@ import decimal import logging import re +from collections import defaultdict from pathlib import Path from typing import Any, Optional @@ -32,6 +33,12 @@ class Book: + # Need to track state of duplicate deposit/withdrawal entries + # All deposits/withdrawals are held back until they occur a second time + # Initialize non-existing fields with None once they're called + kraken_held_ops: defaultdict[str, defaultdict[str, Any]] = \ + defaultdict(lambda: defaultdict(lambda: None)) + def __init__(self, price_data: PriceData) -> None: self.price_data = price_data @@ -458,10 +465,6 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: "withdrawal": "Withdrawal", } - # Need to track state of duplicate deposit/withdrawal entries - # All deposits/withdrawals are held back until they occur a second time - held_operations: list[dict[str, Any]] = [] - with open(file_path, encoding="utf8") as f: reader = csv.reader(f) @@ -574,73 +577,79 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: # in the public trade history and are skipped. # For staking / unstaking / staking reward actions, deposits / # withdrawals only occur once and will be ignored. + # The "appended" flag stores if an operation for a given refid has + # already been appended to the operations list: + # == None: Initial value, this is the first occurence + # == False: No operation has been appended, this is the second occurene + # == True: Operation has already been appended, this should not happen if operation in ["Deposit", "Withdrawal"]: - # search for refid in refids list - refid_idxs = [ - idx - for idx, op in enumerate(held_operations) - if op["refid"] == refid - ] - # refid should not exist more than once in list - if len(refid_idxs) > 1: - log.error( - f"{file_path} row {row}: More than two entries with refid " - f"{refid} should not exist ({operation}). " - "Please create an Issue or PR." - ) - raise RuntimeError - # if refid already exists, assert that data of operations agree - if refid_idxs: - idx = refid_idxs[0] + # If this is the first occurence, set the "appended" flag to false + # and don't append the operation to the list. Instead, store the + # data for verifying or appending it later. + if self.kraken_held_ops[refid]["appended"] is None: + append_operation = False + self.kraken_held_ops[refid]["appended"] = False + self.kraken_held_ops[refid]["operation"] = operation + self.kraken_held_ops[refid]["utc_time"] = utc_time + self.kraken_held_ops[refid]["platform"] = platform + self.kraken_held_ops[refid]["change"] = change + self.kraken_held_ops[refid]["coin"] = coin + self.kraken_held_ops[refid]["row"] = row + self.kraken_held_ops[refid]["file_path"] = file_path + self.kraken_held_ops[refid]["fee"] = fee + # If this is the second occurence, append a new operation, set the + # "appended" flag to True and assert that the data of this operation + # agrees with the data of the first occurence. + elif self.kraken_held_ops[refid]["appended"] is False: + append_operation = True + self.kraken_held_ops[refid]["appended"] = True try: assert ( - operation == held_operations[idx]["operation"] + operation == self.kraken_held_ops[refid]["operation"] ), "operation" - assert change == held_operations[idx]["change"], "change" - assert coin == held_operations[idx]["coin"], "coin" + assert ( + change == self.kraken_held_ops[refid]["change"] + ), "change" + assert coin == self.kraken_held_ops[refid]["coin"], "coin" except AssertionError as e: log.error( f"{file_path} row {row}: Parameters for refid {refid} " - f"({operation}) do not agree: {e} " + f"({operation}) do not agree: {e}. " "Please create an Issue or PR." ) raise RuntimeError - # add all entries to refid list and held operations list - held_operations.append( - { - "refid": refid, - "operation": operation, - "utc_time": utc_time, - "platform": platform, - "change": change, - "coin": coin, - "row": row, - "file_path": file_path, - "fee": fee, - } - ) + # If an operation with the same refid has been already appended, + # this is the third occurence. Throw an error if this happens. + elif self.kraken_held_ops[refid]["appended"] is True: + log.error( + f"{file_path} row {row}: More than two entries with refid " + f"{refid} should not exist ({operation}). " + "Please create an Issue or PR." + ) + raise RuntimeError + # This should never happen + else: + log.error( + f"{file_path} row {row}: Unknown value for appended " + f"operation flag {self.kraken_held_ops[refid]['appended']}." + "Please create an Issue or PR." + ) + raise RuntimeError - if operation == "Deposit": - # append only the second deposit to the operations list - append_operation = len(refid_idxs) == 1 - elif operation == "Withdrawal": - # Append the first withdrawal to the operations list as soon - # as the second withdrawal occurs. Therefore, for the second - # withdrawal, overwrite the variables with the data of the first - # withdrawal and append this to the operations list. - if refid_idxs: - append_operation = True - # required for type annotation: convert Optional[str] to str - operation = str(held_operations[idx]["operation"]) - utc_time = held_operations[idx]["utc_time"] - platform = held_operations[idx]["platform"] - change = held_operations[idx]["change"] - coin = held_operations[idx]["coin"] - row = held_operations[idx]["row"] - file_path = held_operations[idx]["file_path"] - fee = held_operations[idx]["fee"] - else: - append_operation = False + # For deposits, this is all we need to do. + # For withdrawals, we need to append the first withdrawal as soon as + # the second withdrawal occurs. Therefore, overwrite the variables + # with the data of the first withdrawal and append it. + if append_operation and operation == "Withdrawal": + # required for type annotation: convert Optional[str] to str + operation = str(self.kraken_held_ops[refid]["operation"]) + utc_time = self.kraken_held_ops[refid]["utc_time"] + platform = self.kraken_held_ops[refid]["platform"] + change = self.kraken_held_ops[refid]["change"] + coin = self.kraken_held_ops[refid]["coin"] + row = self.kraken_held_ops[refid]["row"] + file_path = self.kraken_held_ops[refid]["file_path"] + fee = self.kraken_held_ops[refid]["fee"] if append_operation: self.append_operation( From 2385dfe86ec97a9b51b7e1b69c2770cf82c6985c Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Sun, 20 Feb 2022 14:37:25 +0100 Subject: [PATCH 20/26] ADD create_operation and append_created_operation --- src/book.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/book.py b/src/book.py index 8a795854..e3559029 100644 --- a/src/book.py +++ b/src/book.py @@ -47,6 +47,38 @@ def __init__(self, price_data: PriceData) -> None: def __bool__(self) -> bool: return bool(self.operations) + def create_operation( + self, + operation: str, + utc_time: datetime.datetime, + platform: str, + change: decimal.Decimal, + coin: str, + row: int, + file_path: Path, + ) -> Optional[tr.Operation]: + + try: + Op = getattr(tr, operation) + except AttributeError: + log.warning( + "Could not recognize operation `%s` in %s file `%s:%i`.", + operation, + platform, + file_path, + row, + ) + return None + + return Op(utc_time, platform, change, coin, row, file_path) + + def append_created_operation( + self, + operation: tr.Operation, + ) -> None: + + self.operations.append(operation) + def append_operation( self, operation: str, From f461f5f738e030419a61923b08412f7d7e99051b Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Sun, 20 Feb 2022 14:37:58 +0100 Subject: [PATCH 21/26] REFACTOR handling of deposits/withdrawals based on stored operation classes --- src/book.py | 72 ++++++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/src/book.py b/src/book.py index e3559029..a555c70f 100644 --- a/src/book.py +++ b/src/book.py @@ -596,10 +596,6 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: assert coin assert change - # append all operations to main list per default - # will be overwritten for deposits / withdrawals - append_operation = True - # Skip duplicate entries for deposits / withdrawals and additional # deposit / withdrawal lines for staking / unstaking / staking reward # actions. @@ -612,37 +608,45 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: # The "appended" flag stores if an operation for a given refid has # already been appended to the operations list: # == None: Initial value, this is the first occurence - # == False: No operation has been appended, this is the second occurene + # == False: No operation has been appended, this is the second occurence # == True: Operation has already been appended, this should not happen if operation in ["Deposit", "Withdrawal"]: + # First, create the operations + op = self.create_operation( + operation, utc_time, platform, change, coin, row, file_path + ) + op_fee = None + if fee != 0: + op_fee = self.create_operation( + "Fee", utc_time, platform, fee, coin, row, file_path + ) # If this is the first occurence, set the "appended" flag to false # and don't append the operation to the list. Instead, store the # data for verifying or appending it later. if self.kraken_held_ops[refid]["appended"] is None: - append_operation = False self.kraken_held_ops[refid]["appended"] = False - self.kraken_held_ops[refid]["operation"] = operation - self.kraken_held_ops[refid]["utc_time"] = utc_time - self.kraken_held_ops[refid]["platform"] = platform - self.kraken_held_ops[refid]["change"] = change - self.kraken_held_ops[refid]["coin"] = coin - self.kraken_held_ops[refid]["row"] = row - self.kraken_held_ops[refid]["file_path"] = file_path - self.kraken_held_ops[refid]["fee"] = fee + self.kraken_held_ops[refid]["operation"] = op + self.kraken_held_ops[refid]["operation_fee"] = op_fee # If this is the second occurence, append a new operation, set the # "appended" flag to True and assert that the data of this operation # agrees with the data of the first occurence. elif self.kraken_held_ops[refid]["appended"] is False: - append_operation = True self.kraken_held_ops[refid]["appended"] = True try: assert ( - operation == self.kraken_held_ops[refid]["operation"] + isinstance( + type(op), + type(self.kraken_held_ops[refid]["operation"]) + ) ), "operation" assert ( - change == self.kraken_held_ops[refid]["change"] + op.change == \ + self.kraken_held_ops[refid]["operation"].change ), "change" - assert coin == self.kraken_held_ops[refid]["coin"], "coin" + assert ( + op.coin == \ + self.kraken_held_ops[refid]["operation"].coin + ), "coin" except AssertionError as e: log.error( f"{file_path} row {row}: Parameters for refid {refid} " @@ -650,6 +654,20 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: "Please create an Issue or PR." ) raise RuntimeError + # For deposits, this is all we need to do before appending the + # operation. For withdrawals, we need to append the first + # withdrawal as soon as the second withdrawal occurs. Therefore, + # overwrite the operation with the stored first withdrawal. + if operation == "Withdrawal": + op = self.kraken_held_ops[refid]["operation"] + op_fee = self.kraken_held_ops[refid]["operation_fee"] + # Finally, append the operations and delete the stored + # operations to reduce memory consumption + self.append_created_operation(op) + if op_fee: + self.append_created_operation(op_fee) + self.kraken_held_ops[refid]["operation"] = None + self.kraken_held_ops[refid]["operation_fee"] = None # If an operation with the same refid has been already appended, # this is the third occurence. Throw an error if this happens. elif self.kraken_held_ops[refid]["appended"] is True: @@ -668,22 +686,8 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: ) raise RuntimeError - # For deposits, this is all we need to do. - # For withdrawals, we need to append the first withdrawal as soon as - # the second withdrawal occurs. Therefore, overwrite the variables - # with the data of the first withdrawal and append it. - if append_operation and operation == "Withdrawal": - # required for type annotation: convert Optional[str] to str - operation = str(self.kraken_held_ops[refid]["operation"]) - utc_time = self.kraken_held_ops[refid]["utc_time"] - platform = self.kraken_held_ops[refid]["platform"] - change = self.kraken_held_ops[refid]["change"] - coin = self.kraken_held_ops[refid]["coin"] - row = self.kraken_held_ops[refid]["row"] - file_path = self.kraken_held_ops[refid]["file_path"] - fee = self.kraken_held_ops[refid]["fee"] - - if append_operation: + # for all other operation types + else: self.append_operation( operation, utc_time, platform, change, coin, row, file_path ) From 99baef51f530cd7a880ab7628ae0f08da390a724 Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Sun, 20 Feb 2022 19:26:24 +0100 Subject: [PATCH 22/26] REFACTOR create_operation, append_operation --- src/book.py | 54 ++++++++++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/src/book.py b/src/book.py index a555c70f..bcdae2a9 100644 --- a/src/book.py +++ b/src/book.py @@ -56,23 +56,25 @@ def create_operation( coin: str, row: int, file_path: Path, - ) -> Optional[tr.Operation]: + ) -> tr.Operation: try: Op = getattr(tr, operation) except AttributeError: - log.warning( + log.error( "Could not recognize operation `%s` in %s file `%s:%i`.", operation, platform, file_path, row, ) - return None + raise RuntimeError - return Op(utc_time, platform, change, coin, row, file_path) + op = Op(utc_time, platform, change, coin, row, file_path) + assert isinstance(op, tr.Operation) + return op - def append_created_operation( + def _append_operation( self, operation: tr.Operation, ) -> None: @@ -90,20 +92,17 @@ def append_operation( file_path: Path, ) -> None: - try: - Op = getattr(tr, operation) - except AttributeError: - log.warning( - "Could not recognize operation `%s` in %s file `%s:%i`.", - operation, - platform, - file_path, - row, - ) - return + op = self.create_operation( + operation, + utc_time, + platform, + change, + coin, + row, + file_path, + ) - o = Op(utc_time, platform, change, coin, row, file_path) - self.operations.append(o) + self._append_operation(op) def _read_binance(self, file_path: Path, version: int = 1) -> None: platform = "binance" @@ -635,17 +634,16 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: try: assert ( isinstance( - type(op), - type(self.kraken_held_ops[refid]["operation"]) + op, type(self.kraken_held_ops[refid]["operation"]) ) ), "operation" assert ( - op.change == \ - self.kraken_held_ops[refid]["operation"].change + op.change + == self.kraken_held_ops[refid]["operation"].change ), "change" assert ( - op.coin == \ - self.kraken_held_ops[refid]["operation"].coin + op.coin + == self.kraken_held_ops[refid]["operation"].coin ), "coin" except AssertionError as e: log.error( @@ -663,11 +661,11 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: op_fee = self.kraken_held_ops[refid]["operation_fee"] # Finally, append the operations and delete the stored # operations to reduce memory consumption - self.append_created_operation(op) + self._append_operation(op) if op_fee: - self.append_created_operation(op_fee) - self.kraken_held_ops[refid]["operation"] = None - self.kraken_held_ops[refid]["operation_fee"] = None + self._append_operation(op_fee) + del(self.kraken_held_ops[refid]["operation"]) + del(self.kraken_held_ops[refid]["operation_fee"]) # If an operation with the same refid has been already appended, # this is the third occurence. Throw an error if this happens. elif self.kraken_held_ops[refid]["appended"] is True: From 09817ed1f746fbc850d1e1ea00de8b19c8645dd0 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 19 Mar 2022 13:58:39 +0100 Subject: [PATCH 23/26] AUTOFORMAT book --- src/book.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/book.py b/src/book.py index bcdae2a9..74c23fec 100644 --- a/src/book.py +++ b/src/book.py @@ -36,8 +36,9 @@ class Book: # Need to track state of duplicate deposit/withdrawal entries # All deposits/withdrawals are held back until they occur a second time # Initialize non-existing fields with None once they're called - kraken_held_ops: defaultdict[str, defaultdict[str, Any]] = \ - defaultdict(lambda: defaultdict(lambda: None)) + kraken_held_ops: defaultdict[str, defaultdict[str, Any]] = defaultdict( + lambda: defaultdict(lambda: None) + ) def __init__(self, price_data: PriceData) -> None: self.price_data = price_data @@ -632,18 +633,15 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: elif self.kraken_held_ops[refid]["appended"] is False: self.kraken_held_ops[refid]["appended"] = True try: - assert ( - isinstance( - op, type(self.kraken_held_ops[refid]["operation"]) - ) + assert isinstance( + op, type(self.kraken_held_ops[refid]["operation"]) ), "operation" assert ( op.change == self.kraken_held_ops[refid]["operation"].change ), "change" assert ( - op.coin - == self.kraken_held_ops[refid]["operation"].coin + op.coin == self.kraken_held_ops[refid]["operation"].coin ), "coin" except AssertionError as e: log.error( @@ -664,8 +662,8 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: self._append_operation(op) if op_fee: self._append_operation(op_fee) - del(self.kraken_held_ops[refid]["operation"]) - del(self.kraken_held_ops[refid]["operation_fee"]) + del self.kraken_held_ops[refid]["operation"] + del self.kraken_held_ops[refid]["operation_fee"] # If an operation with the same refid has been already appended, # this is the third occurence. Throw an error if this happens. elif self.kraken_held_ops[refid]["appended"] is True: From 7a05a5009de42c4e9687a4508d979700583a4220 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 19 Mar 2022 14:05:12 +0100 Subject: [PATCH 24/26] FIX typo --- src/book.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/book.py b/src/book.py index 74c23fec..84521340 100644 --- a/src/book.py +++ b/src/book.py @@ -607,8 +607,8 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: # withdrawals only occur once and will be ignored. # The "appended" flag stores if an operation for a given refid has # already been appended to the operations list: - # == None: Initial value, this is the first occurence - # == False: No operation has been appended, this is the second occurence + # == None: Initial value, this is the first occurrence + # == False: No operation has been appended, this is the second occurrence # == True: Operation has already been appended, this should not happen if operation in ["Deposit", "Withdrawal"]: # First, create the operations @@ -620,16 +620,16 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: op_fee = self.create_operation( "Fee", utc_time, platform, fee, coin, row, file_path ) - # If this is the first occurence, set the "appended" flag to false + # If this is the first occurrence, set the "appended" flag to false # and don't append the operation to the list. Instead, store the # data for verifying or appending it later. if self.kraken_held_ops[refid]["appended"] is None: self.kraken_held_ops[refid]["appended"] = False self.kraken_held_ops[refid]["operation"] = op self.kraken_held_ops[refid]["operation_fee"] = op_fee - # If this is the second occurence, append a new operation, set the + # If this is the second occurrence, append a new operation, set the # "appended" flag to True and assert that the data of this operation - # agrees with the data of the first occurence. + # agrees with the data of the first occurrence. elif self.kraken_held_ops[refid]["appended"] is False: self.kraken_held_ops[refid]["appended"] = True try: @@ -665,7 +665,7 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: del self.kraken_held_ops[refid]["operation"] del self.kraken_held_ops[refid]["operation_fee"] # If an operation with the same refid has been already appended, - # this is the third occurence. Throw an error if this happens. + # this is the third occurrence. Throw an error if this happens. elif self.kraken_held_ops[refid]["appended"] is True: log.error( f"{file_path} row {row}: More than two entries with refid " From 20d521da2a98aa167d4fe9a5475c1a62b8b78a3e Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 19 Mar 2022 14:05:26 +0100 Subject: [PATCH 25/26] CHANGE raise TypeError instead of RuntimeError for wrong type --- src/book.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/book.py b/src/book.py index 84521340..3c651a92 100644 --- a/src/book.py +++ b/src/book.py @@ -680,7 +680,7 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: f"operation flag {self.kraken_held_ops[refid]['appended']}." "Please create an Issue or PR." ) - raise RuntimeError + raise TypeError # for all other operation types else: From 6ccfdc04843817cb30fd046d80d13aa41df2e007 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 19 Mar 2022 14:08:01 +0100 Subject: [PATCH 26/26] UPDATE shorten comment to fit into line --- src/book.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/book.py b/src/book.py index 3c651a92..2e672894 100644 --- a/src/book.py +++ b/src/book.py @@ -607,8 +607,8 @@ def _read_kraken_ledgers(self, file_path: Path) -> None: # withdrawals only occur once and will be ignored. # The "appended" flag stores if an operation for a given refid has # already been appended to the operations list: - # == None: Initial value, this is the first occurrence - # == False: No operation has been appended, this is the second occurrence + # == None: Initial value (first occurrence) + # == False: No operation has been appended (second occurrence) # == True: Operation has already been appended, this should not happen if operation in ["Deposit", "Withdrawal"]: # First, create the operations