From 0ca022c18114d815374093c04b122fbbfec6f980 Mon Sep 17 00:00:00 2001
From: Andrew <47720952+andrewhong5297@users.noreply.github.com>
Date: Sat, 3 Feb 2024 22:06:04 -0500
Subject: [PATCH] add uploads
---
.github/workflows/upload_to_dune.yml | 33 +++
queries/queries.yml | 3 +-
...doswap_v2_collections_summar___2615781.sql | 220 ------------------
queries/sudoswap_v2_trends___2615782.sql | 14 --
requirements.txt | 2 +-
scripts/push_to_dune.py | 17 +-
scripts/upload_to_dune.py | 33 +++
uploads/FEDFUNDS.csv | 206 ++++++++++++++++
8 files changed, 283 insertions(+), 245 deletions(-)
create mode 100644 .github/workflows/upload_to_dune.yml
delete mode 100644 queries/sudoswap_v2_collections_summar___2615781.sql
delete mode 100644 queries/sudoswap_v2_trends___2615782.sql
create mode 100644 scripts/upload_to_dune.py
create mode 100644 uploads/FEDFUNDS.csv
diff --git a/.github/workflows/upload_to_dune.yml b/.github/workflows/upload_to_dune.yml
new file mode 100644
index 0000000..4f079de
--- /dev/null
+++ b/.github/workflows/upload_to_dune.yml
@@ -0,0 +1,33 @@
+name: Upload CSVs to Dune on Commit
+
+on:
+ push:
+ branches:
+ - main
+ paths:
+ - 'queries/**'
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v2
+
+ - uses: actions/setup-python@v4
+ with:
+ python-version: '3.9'
+
+ - name: Log directory structure
+ run: |
+ pwd
+ ls -R
+
+ - name: pip requirements
+ run: pip install -r requirements.txt
+
+ - name: Update all queries from Dune, by overwriting queries with repo query text
+ env:
+ DUNE_API_KEY: ${{ secrets.DUNE_API_KEY }}
+ DUNE_API_BASE_URL: ${{ secrets.DUNE_API_BASE_URL }}
+ run: python -u scripts/upload_to_dune.py
diff --git a/queries/queries.yml b/queries/queries.yml
index 4c240ef..a276af1 100644
--- a/queries/queries.yml
+++ b/queries/queries.yml
@@ -1,3 +1,2 @@
query_ids:
- - 2615782
- - 2615781
\ No newline at end of file
+ - 123456
\ No newline at end of file
diff --git a/queries/sudoswap_v2_collections_summar___2615781.sql b/queries/sudoswap_v2_collections_summar___2615781.sql
deleted file mode 100644
index a1c16bc..0000000
--- a/queries/sudoswap_v2_collections_summar___2615781.sql
+++ /dev/null
@@ -1,220 +0,0 @@
--- part of a query repo
--- query name: Sudoswap V2 Collections Summary
--- query link: https://dune.com/queries/2615781
-
-
-WITH
- pairs_created as (
- SELECT
- *
- FROM query_2615780
- ),
-
- --we also need to get token balance of each pair currently
- erc721_balances as (
- SELECT
- contract_address as nft_contract_address
- -- , array_agg(tokenId) as tokenids_held
- , count(*) as tokens_held
- FROM (
- SELECT
- row_number() OVER (partition by contract_address, tokenId order by evt_block_number desc, evt_index desc) as last_held
- , *
- FROM erc721_ethereum.evt_Transfer tr
- where tr.contract_address IN (select distinct nft_contract_address from pairs_created) --get last holders for all nft contracts (need to include erc1155 later)
- ) a
- WHERE last_held = 1
- AND a.to IN (SELECT pool_address FROM pairs_created) --keep only tokens held by pairs
- GROUP BY 1
- ),
-
- erc1155_balances as (
- with t_in as (
- SELECT
- to as pool
- , contract_address
- , token_id
- , sum(amount) as transferred_in
- FROM nft.transfers
- WHERE blockchain = 'ethereum'
- and token_standard = 'erc1155'
- and to IN (SELECT pool_address FROM pairs_created)
- group by 1,2,3
- )
-
- , t_out as (
- SELECT
- "from" as pool
- , contract_address
- , token_id
- , sum(amount) as transferred_out
- FROM nft.transfers
- WHERE blockchain = 'ethereum'
- and token_standard = 'erc1155'
- and "from" IN (SELECT pool_address FROM pairs_created)
- group by 1,2,3
- )
-
- SELECT
- pc.nft_contract_address
- , sum(t_in.transferred_in - COALESCE(t_out.transferred_out,cast(0 as uint256))) as tokens_held
- FROM t_in
- LEFT JOIN t_out ON t_in.pool = t_out.pool AND t_in.contract_address = t_out.contract_address AND t_in.token_id = t_out.token_id
- JOIN pairs_created pc ON pc.pool_address = t_in.pool
- group by 1
- ),
-
- --we need eth_balances for liquidity tracking purposes
- eth_balances as (
- WITH eth_in as (
- SELECT
- tr.to as holder_address
- , SUM(tr.value/1e18) as eth_funded
- FROM ethereum.traces tr
- WHERE tr.block_time > timestamp '2022-04-23'
- AND tr.success=true
- AND tr.type='call'
- AND (tr.call_type NOT IN ('delegatecall', 'callcode', 'staticcall') OR tr.call_type IS null)
- AND tr.to IN (SELECT pool_address FROM pairs_created)
- GROUP BY 1
- ),
-
- eth_out as (
- SELECT
- tr."from" as holder_address
- , SUM(tr.value/1e18) as eth_spent
- FROM ethereum.traces tr
- WHERE tr.block_time > timestamp '2022-04-23'
- AND tr.success=true
- AND tr.type='call'
- AND (tr.call_type NOT IN ('delegatecall', 'callcode', 'staticcall') OR tr.call_type IS null)
- AND tr."from" IN (SELECT pool_address FROM pairs_created)
- GROUP BY 1
- )
-
- SELECT
- pc.nft_contract_address
- , SUM(COALESCE(eth_funded,0) - COALESCE(eth_spent, 0)) as eth_balance
- FROM eth_in
- LEFT JOIN eth_out ON eth_in.holder_address = eth_out.holder_address
- JOIN pairs_created pc ON pc.pool_address = eth_in.holder_address
- WHERE COALESCE(eth_funded,0) - COALESCE(eth_spent, 0) > 0 --for some reason some balances are calculated negative.
- GROUP BY 1
- ),
-
- erc20_balances as (
- WITH erc20_in as (
- SELECT
- tr.to as holder
- , contract_address
- , SUM(cast(tr.value as double)) as token_funded
- FROM erc20_ethereum.evt_Transfer tr
- WHERE tr.to IN (SELECT pool_address FROM pairs_created)
- -- AND contract_address IN (SELECT address FROM valid_tokens)
- GROUP BY 1,2
- ),
-
- erc20_out as (
- SELECT
- tr."from" as holder
- , contract_address
- , SUM(cast(tr.value as double)) as token_spent
- FROM erc20_ethereum.evt_Transfer tr
- WHERE tr."from" IN (SELECT pool_address FROM pairs_created)
- -- AND contract_address IN (SELECT address FROM valid_tokens)
- GROUP BY 1,2
- )
-
- , contract_balance_sum as (
- SELECT
- pc.nft_contract_address
- , tk.symbol as symbol
- , erc20_in.contract_address
- , sum(token_funded/pow(10,COALESCE(tk.decimals,18)) - COALESCE(token_spent, 0)/pow(10,COALESCE(tk.decimals,18))) as tokens_held
- FROM erc20_in
- LEFT JOIN erc20_out ON erc20_in.holder = erc20_out.holder
- LEFT JOIN tokens.erc20 tk ON tk.contract_address = erc20_in.contract_address
- JOIN pairs_created pc ON pc.pool_address = erc20_in.holder
- WHERE COALESCE(token_funded,0) - COALESCE(token_spent, 0) > 0 --due to overflow, some balances are calculated negative.
- AND blockchain = 'ethereum'
- group by 1,2,3
- )
-
- SELECT
- nft_contract_address
- , array_agg(json_object('token':COALESCE(symbol,cast(contract_address as varchar)), 'balance': round(tokens_held,4))) as tokens_held
- FROM contract_balance_sum
- group by 1
- ),
-
- trading_totals as (
- SELECT
- nft_contract_address
- , nft_name
- , sum(amount_usd) as usd_volume
- -- get 7 day volume
- , sum(number_of_items) as nfts_traded
- , sum(trade_fee_amount_usd) as trade_fee_amount_usd
- , sum(case when block_time >= now() - interval '7' day then amount_usd else 0 end) as last_7_days
- , sum(case when block_time >= now() - interval '7' day then trade_fee_amount_usd else 0 end) as last_7_days_fees
- , sum(protocol_fee_amount_usd) as protocol_fee_amount_usd
- FROM dune.dune.result_sudoswap_v_2_trades
- GROUP BY 1,2
- ),
-
- last_price as (
- SELECT
- nft_contract_address
- , amount_usd
- FROM (
- SELECT
- *
- , row_number() OVER (partition by nft_contract_address order by block_time desc) as rn
- FROM dune.dune.result_sudoswap_v_2_trades
- WHERE amount_usd is not null
- ) a
- WHERE rn = 1
- ),
-
- all_collections_cleaned as (
- SELECT
- pc.*
- , nft.name as nft_name
- , COALESCE(bal_721.tokens_held,0) as nfts_721 --buyable liquidity
- , COALESCE(bal_1155.tokens_held,cast(0 as uint256)) as nfts_1155 --buyable liquidity
- , COALESCE(bal_20.tokens_held,array['0']) as erc20_balances --fix logic using transform agg later
- , COALESCE(eth_bal.eth_balance,0) as eth_liq --sellable liquidity
- FROM (
- SELECT
- nft_contract_address
- , count(distinct pool_address) as num_pairs
- FROM pairs_created
- GROUP BY 1
- ) pc
- LEFT JOIN erc721_balances bal_721 ON bal_721.nft_contract_address = pc.nft_contract_address
- LEFT JOIN erc1155_balances bal_1155 ON bal_1155.nft_contract_address = pc.nft_contract_address
- LEFT JOIN erc20_balances bal_20 ON bal_20.nft_contract_address = pc.nft_contract_address
- LEFT JOIN eth_balances eth_bal ON eth_bal.nft_contract_address = pc.nft_contract_address
- LEFT JOIN tokens.nft nft ON nft.blockchain = 'ethereum' and nft.contract_address = pc.nft_contract_address
- )
-
-SELECT
- CONCAT(' swap now! ') as s_link
- , CONCAT('',COALESCE(acc.nft_name, cast(acc.nft_contract_address as varchar),'')) as collection
- , num_pairs
- , '||' as split
- , spot.amount_usd as last_spot
- , COALESCE(trade.last_7_days, 0) as last_7_days
- , COALESCE(trade.usd_volume, 0) as usd_volume
- , COALESCE(trade.nfts_traded, cast(0 as uint256)) as nfts_traded
- , COALESCE(trade.trade_fee_amount_usd, 0) as trade_fee_amount_usd
- , COALESCE(trade.protocol_fee_amount_usd, 0) as protocol_fee_amount_usd
- , '||' as split_2
- , nfts_721
- , nfts_1155
- , erc20_balances
- , eth_liq
-FROM all_collections_cleaned acc
-LEFT JOIN trading_totals trade ON trade.nft_contract_address = acc.nft_contract_address
-LEFT JOIN last_price spot ON spot.nft_contract_address = acc.nft_contract_address
-ORDER BY last_7_days DESC
\ No newline at end of file
diff --git a/queries/sudoswap_v2_trends___2615782.sql b/queries/sudoswap_v2_trends___2615782.sql
deleted file mode 100644
index 074a982..0000000
--- a/queries/sudoswap_v2_trends___2615782.sql
+++ /dev/null
@@ -1,14 +0,0 @@
--- part of a query repo
--- query name: Sudoswap V2 Trends
--- query link: https://dune.com/queries/2615782
-
-
-SELECT
-date_trunc('week', block_time) as week
-, pool_type as col
-, count(*) as trades
-, sum(amount_usd) as usd_volume
-, count(distinct tx_from) as traders
-, sum(protocol_fee_amount_usd + trade_fee_amount_usd + royalty_fee_amount_usd) as all_fees
-FROM dune.dune.result_sudoswap_v_2_trades
-group by 1,2
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index ad76048..6888d99 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-dune-client==1.3.0
+dune-client
pyyaml
python-dotenv
pandas
\ No newline at end of file
diff --git a/scripts/push_to_dune.py b/scripts/push_to_dune.py
index 71b1932..86357e2 100644
--- a/scripts/push_to_dune.py
+++ b/scripts/push_to_dune.py
@@ -36,13 +36,14 @@
with open(file_path, 'r', encoding='utf-8') as file:
text = file.read()
- # Update existing file
- dune.update_query(
- query.base.query_id,
- # All parameters below are optional
- query_sql=text,
- )
- print('SUCCESS: updated query {} to dune'.format(query.base.query_id))
-
+ try:
+ # Update existing file
+ dune.update_query(
+ query.base.query_id,
+ query_sql=text,
+ )
+ print('SUCCESS: updated query {} to dune'.format(query.base.query_id))
+ except Exception as e:
+ print('ERROR: {}'.format(str(e))) #likely API permission errors
else:
print('ERROR: file not found, query id {}'.format(query.base.query_id))
diff --git a/scripts/upload_to_dune.py b/scripts/upload_to_dune.py
new file mode 100644
index 0000000..5ec4e36
--- /dev/null
+++ b/scripts/upload_to_dune.py
@@ -0,0 +1,33 @@
+import os
+from dune_client.client import DuneClient
+from dotenv import load_dotenv
+import sys
+import codecs
+import os
+
+# Set the default encoding to UTF-8
+sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
+
+dotenv_path = os.path.join(os.path.dirname(__file__), '..', '.env')
+load_dotenv(dotenv_path)
+
+dune = DuneClient.from_env()
+
+uploads_path = os.path.join(os.path.dirname(__file__), '..', 'uploads')
+files = os.listdir(uploads_path)
+
+for file in files:
+ if not file.endswith(".csv"):
+ continue
+ file_name = file.split(".")[0].lower().replace(' ', '_')
+ with open(os.path.join(uploads_path, file), 'r') as file:
+ try:
+ table = dune.upload_csv(
+ data=str(file.read()),
+ table_name=file_name,
+ is_private=False
+ )
+ print(f'uploaded table "{file_name}"')
+ except Exception as e:
+ print(f"An error occurred while uploading the CSV: {str(e)}")
+
diff --git a/uploads/FEDFUNDS.csv b/uploads/FEDFUNDS.csv
new file mode 100644
index 0000000..d063ea3
--- /dev/null
+++ b/uploads/FEDFUNDS.csv
@@ -0,0 +1,206 @@
+DATE,FEDFUNDS
+2007-01-01,5.25
+2007-02-01,5.26
+2007-03-01,5.26
+2007-04-01,5.25
+2007-05-01,5.25
+2007-06-01,5.25
+2007-07-01,5.26
+2007-08-01,5.02
+2007-09-01,4.94
+2007-10-01,4.76
+2007-11-01,4.49
+2007-12-01,4.24
+2008-01-01,3.94
+2008-02-01,2.98
+2008-03-01,2.61
+2008-04-01,2.28
+2008-05-01,1.98
+2008-06-01,2.00
+2008-07-01,2.01
+2008-08-01,2.00
+2008-09-01,1.81
+2008-10-01,0.97
+2008-11-01,0.39
+2008-12-01,0.16
+2009-01-01,0.15
+2009-02-01,0.22
+2009-03-01,0.18
+2009-04-01,0.15
+2009-05-01,0.18
+2009-06-01,0.21
+2009-07-01,0.16
+2009-08-01,0.16
+2009-09-01,0.15
+2009-10-01,0.12
+2009-11-01,0.12
+2009-12-01,0.12
+2010-01-01,0.11
+2010-02-01,0.13
+2010-03-01,0.16
+2010-04-01,0.20
+2010-05-01,0.20
+2010-06-01,0.18
+2010-07-01,0.18
+2010-08-01,0.19
+2010-09-01,0.19
+2010-10-01,0.19
+2010-11-01,0.19
+2010-12-01,0.18
+2011-01-01,0.17
+2011-02-01,0.16
+2011-03-01,0.14
+2011-04-01,0.10
+2011-05-01,0.09
+2011-06-01,0.09
+2011-07-01,0.07
+2011-08-01,0.10
+2011-09-01,0.08
+2011-10-01,0.07
+2011-11-01,0.08
+2011-12-01,0.07
+2012-01-01,0.08
+2012-02-01,0.10
+2012-03-01,0.13
+2012-04-01,0.14
+2012-05-01,0.16
+2012-06-01,0.16
+2012-07-01,0.16
+2012-08-01,0.13
+2012-09-01,0.14
+2012-10-01,0.16
+2012-11-01,0.16
+2012-12-01,0.16
+2013-01-01,0.14
+2013-02-01,0.15
+2013-03-01,0.14
+2013-04-01,0.15
+2013-05-01,0.11
+2013-06-01,0.09
+2013-07-01,0.09
+2013-08-01,0.08
+2013-09-01,0.08
+2013-10-01,0.09
+2013-11-01,0.08
+2013-12-01,0.09
+2014-01-01,0.07
+2014-02-01,0.07
+2014-03-01,0.08
+2014-04-01,0.09
+2014-05-01,0.09
+2014-06-01,0.10
+2014-07-01,0.09
+2014-08-01,0.09
+2014-09-01,0.09
+2014-10-01,0.09
+2014-11-01,0.09
+2014-12-01,0.12
+2015-01-01,0.11
+2015-02-01,0.11
+2015-03-01,0.11
+2015-04-01,0.12
+2015-05-01,0.12
+2015-06-01,0.13
+2015-07-01,0.13
+2015-08-01,0.14
+2015-09-01,0.14
+2015-10-01,0.12
+2015-11-01,0.12
+2015-12-01,0.24
+2016-01-01,0.34
+2016-02-01,0.38
+2016-03-01,0.36
+2016-04-01,0.37
+2016-05-01,0.37
+2016-06-01,0.38
+2016-07-01,0.39
+2016-08-01,0.40
+2016-09-01,0.40
+2016-10-01,0.40
+2016-11-01,0.41
+2016-12-01,0.54
+2017-01-01,0.65
+2017-02-01,0.66
+2017-03-01,0.79
+2017-04-01,0.90
+2017-05-01,0.91
+2017-06-01,1.04
+2017-07-01,1.15
+2017-08-01,1.16
+2017-09-01,1.15
+2017-10-01,1.15
+2017-11-01,1.16
+2017-12-01,1.30
+2018-01-01,1.41
+2018-02-01,1.42
+2018-03-01,1.51
+2018-04-01,1.69
+2018-05-01,1.70
+2018-06-01,1.82
+2018-07-01,1.91
+2018-08-01,1.91
+2018-09-01,1.95
+2018-10-01,2.19
+2018-11-01,2.20
+2018-12-01,2.27
+2019-01-01,2.40
+2019-02-01,2.40
+2019-03-01,2.41
+2019-04-01,2.42
+2019-05-01,2.39
+2019-06-01,2.38
+2019-07-01,2.40
+2019-08-01,2.13
+2019-09-01,2.04
+2019-10-01,1.83
+2019-11-01,1.55
+2019-12-01,1.55
+2020-01-01,1.55
+2020-02-01,1.58
+2020-03-01,0.65
+2020-04-01,0.05
+2020-05-01,0.05
+2020-06-01,0.08
+2020-07-01,0.09
+2020-08-01,0.10
+2020-09-01,0.09
+2020-10-01,0.09
+2020-11-01,0.09
+2020-12-01,0.09
+2021-01-01,0.09
+2021-02-01,0.08
+2021-03-01,0.07
+2021-04-01,0.07
+2021-05-01,0.06
+2021-06-01,0.08
+2021-07-01,0.10
+2021-08-01,0.09
+2021-09-01,0.08
+2021-10-01,0.08
+2021-11-01,0.08
+2021-12-01,0.08
+2022-01-01,0.08
+2022-02-01,0.08
+2022-03-01,0.20
+2022-04-01,0.33
+2022-05-01,0.77
+2022-06-01,1.21
+2022-07-01,1.68
+2022-08-01,2.33
+2022-09-01,2.56
+2022-10-01,3.08
+2022-11-01,3.78
+2022-12-01,4.10
+2023-01-01,4.33
+2023-02-01,4.57
+2023-03-01,4.65
+2023-04-01,4.83
+2023-05-01,5.06
+2023-06-01,5.08
+2023-07-01,5.12
+2023-08-01,5.33
+2023-09-01,5.33
+2023-10-01,5.33
+2023-11-01,5.33
+2023-12-01,5.33
+2024-01-01,5.33