From 174c272dd89bb9f9260535d00a2b6a0858bc485a Mon Sep 17 00:00:00 2001 From: Peanut Date: Mon, 17 Jul 2023 11:10:47 +0800 Subject: [PATCH] feat: add firefox usage and example (#51) * feat: add example Signed-off-by: vvanglro * feat: add example Signed-off-by: vvanglro * feat: update every_day_cron_challenge.yml Signed-off-by: vvanglro * feat: update README.md Signed-off-by: vvanglro * feat: update pre-commit Signed-off-by: vvanglro --------- Signed-off-by: vvanglro --- .../workflows/every_day_cron_challenge.yml | 6 +- .pre-commit-config.yaml | 3 +- README.md | 130 +++++------------- example/__init__.py | 0 .../test_chromium_cron_challenge.py | 17 ++- example/test_firefox_cron_challenge.py | 45 ++++++ 6 files changed, 97 insertions(+), 104 deletions(-) create mode 100644 example/__init__.py rename tests/test_cron_challenge.py => example/test_chromium_cron_challenge.py (59%) create mode 100644 example/test_firefox_cron_challenge.py diff --git a/.github/workflows/every_day_cron_challenge.yml b/.github/workflows/every_day_cron_challenge.yml index 80599ef..d801eb5 100644 --- a/.github/workflows/every_day_cron_challenge.yml +++ b/.github/workflows/every_day_cron_challenge.yml @@ -19,7 +19,9 @@ jobs: run: | python -m pip install --upgrade pdm pdm install - pdm run playwright install chromium + pdm run playwright install chromium firefox sudo apt install xvfb - name: Run script - run: xvfb-run pdm run python tests/test_cron_challenge.py + run: | + xvfb-run pdm run python example/test_chromium_cron_challenge.py + pdm run python example/test_firefox_cron_challenge.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 45d8d0a..c0a2c8b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,6 +5,7 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml + - id: check-toml - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.0.277 @@ -32,4 +33,4 @@ repos: entry: '\bprint\(' language: pygrep types: [ python ] - exclude: "tests/test_cron_challenge.py" + exclude: "example/test_chromium_cron_challenge.py|example/test_firefox_cron_challenge.py" diff --git a/README.md b/README.md index ae58cf1..a924260 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,11 @@ the cf_clearance, make sure you use the same IP and UA as when you got it. ## Warning -Please use interface mode, You must add headless=False. +If you use chromium, Please use interface mode, You must add headless=False. If you use it on linux or docker, use XVFB. +If you use firefox you don't need interface mode and XVFB. + Challenge are not always successful. Please try more and handle exceptions. @@ -39,22 +41,39 @@ curl http://localhost:8000/challenge -H "Content-Type:application/json" -X POST import requests proxy = "socks5://localhost:7890" -resp = requests.post("http://localhost:8000/challenge", - json={"proxy": {"server": proxy}, "timeout": 20, - "url": "https://nowsecure.nl"}) +resp = requests.post( + "http://localhost:8000/challenge", + json={ + "proxy": {"server": proxy}, + "timeout": 20, + "url": "https://nowsecure.nl", + "pure": True, + "browser": 2, + "cookies": [ + { + "url": "https://www.example.com", + "name": "example-cookie", + "value": "example-value", + } + ], + "headers": {"example-ua": "example-ua-value"}, + "exec_js": "() => {return navigator.userAgent}", + }, +) data = resp.json() # In some cases, the cloudflare challenge will not be triggered, # so when cf in the return parameter is true, it means that the challenge has been encountered. if data.get("success") and data.get("cf"): ua = data.get("user_agent") + exec_js_resp = data.get("exec_js_resp") cf_clearance_value = data.get("cookies").get("cf_clearance") # use cf_clearance, must be same IP and UA headers = {"user-agent": ua} cookies = {"cf_clearance": cf_clearance_value} - res = requests.get('https://nowsecure.nl', proxies={ - "all": proxy - }, headers=headers, cookies=cookies) - if 'Just a moment...' not in res.text: + res = requests.get( + "https://nowsecure.nl", proxies={"all": proxy}, headers=headers, cookies=cookies + ) + if "Just a moment..." not in res.text: print("cf challenge success") ``` @@ -66,7 +85,7 @@ pip install cf-clearance ## Usage -Please make sure it is the latest package. +Please make sure it is the latest package. See [example](https://github.com/vvanglro/cf-clearance/tree/main/example). ``` pip install --upgrade cf-clearance @@ -76,94 +95,7 @@ or pip install git+https://github.com/vvanglro/cf-clearance.git@main ``` -### sync - -```python -from playwright.sync_api import sync_playwright -from cf_clearance import sync_cf_retry, sync_stealth -import requests - -# not use cf_clearance, cf challenge is fail -proxies = { - "all": "socks5://localhost:7890" -} -res = requests.get('https://nowsecure.nl', proxies=proxies) -if 'Just a moment...' in res.text: - print("cf challenge fail") -# get cf_clearance -with sync_playwright() as p: - browser = p.chromium.launch(headless=False, proxy={"server": "socks5://localhost:7890"}) - page = browser.new_page() - sync_stealth(page, pure=True) - page.goto('https://nowsecure.nl') - res, cf = sync_cf_retry(page) - if cf: - if res: - cookies = page.context.cookies() - for cookie in cookies: - if cookie.get('name') == 'cf_clearance': - cf_clearance_value = cookie.get('value') - print(cf_clearance_value) - ua = page.evaluate('() => {return navigator.userAgent}') - print(ua) - else: - print("cf challenge fail") - else: - print("No cloudflare challenges encountered") - browser.close() -# use cf_clearance, must be same IP and UA -headers = {"user-agent": ua} -cookies = {"cf_clearance": cf_clearance_value} -res = requests.get('https://nowsecure.nl', proxies=proxies, headers=headers, cookies=cookies) -if 'Just a moment...' not in res.text: - print("cf challenge success") -``` - -### async - -```python -import asyncio -from playwright.async_api import async_playwright -from cf_clearance import async_cf_retry, async_stealth -import requests - - -async def main(): - # not use cf_clearance, cf challenge is fail - proxies = { - "all": "socks5://localhost:7890" - } - res = requests.get('https://nowsecure.nl', proxies=proxies) - if 'Just a moment...' in res.text: - print("cf challenge fail") - # get cf_clearance - async with async_playwright() as p: - browser = await p.chromium.launch(headless=False, proxy={"server": "socks5://localhost:7890"}) - page = await browser.new_page() - await async_stealth(page, pure=True) - await page.goto('https://nowsecure.nl') - res, cf = await async_cf_retry(page) - if cf: - if res: - cookies = await page.context.cookies() - for cookie in cookies: - if cookie.get('name') == 'cf_clearance': - cf_clearance_value = cookie.get('value') - print(cf_clearance_value) - ua = await page.evaluate('() => {return navigator.userAgent}') - print(ua) - else: - print("cf challenge fail") - else: - print("No cloudflare challenges encountered") - await browser.close() - # use cf_clearance, must be same IP and UA - headers = {"user-agent": ua} - cookies = {"cf_clearance": cf_clearance_value} - res = requests.get('https://nowsecure.nl', proxies=proxies, headers=headers, cookies=cookies) - if 'Just a moment...' not in res.text: - print("cf challenge success") - - -asyncio.get_event_loop().run_until_complete(main()) +## Install Playwright Depends +```shell +playwright install chromium firefox ``` diff --git a/example/__init__.py b/example/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_cron_challenge.py b/example/test_chromium_cron_challenge.py similarity index 59% rename from tests/test_cron_challenge.py rename to example/test_chromium_cron_challenge.py index d3e0858..24e6203 100644 --- a/tests/test_cron_challenge.py +++ b/example/test_chromium_cron_challenge.py @@ -1,9 +1,14 @@ +import requests from playwright.async_api import async_playwright from cf_clearance import async_cf_retry, async_stealth async def test_cf_challenge(url: str): + # not use cf_clearance, cf challenge is fail + res = requests.get("https://nowsecure.nl") + assert "Just a moment..." in res.text + # get cf_clearance async with async_playwright() as p: browser = await p.chromium.launch( headless=False, @@ -12,19 +17,27 @@ async def test_cf_challenge(url: str): page = await context.new_page() await async_stealth(page, pure=True) await page.goto(url) - res, cf = await async_cf_retry(page) + success, cf = await async_cf_retry(page) if cf: - if res: + if success: cookies = await page.context.cookies() for cookie in cookies: if cookie.get("name") == "cf_clearance": cf_clearance_value = cookie.get("value") + print(cf_clearance_value) + ua = await page.evaluate("() => {return navigator.userAgent}") assert cf_clearance_value else: raise else: print("No cloudflare challenges encountered") + await browser.close() + # use cf_clearance, must be same IP and UA + headers = {"user-agent": ua} + cookies = {"cf_clearance": cf_clearance_value} + res = requests.get("https://nowsecure.nl", headers=headers, cookies=cookies) + assert "Just a moment..." not in res.text if __name__ == "__main__": diff --git a/example/test_firefox_cron_challenge.py b/example/test_firefox_cron_challenge.py new file mode 100644 index 0000000..ce2ffd8 --- /dev/null +++ b/example/test_firefox_cron_challenge.py @@ -0,0 +1,45 @@ +import requests +from playwright.async_api import async_playwright + +from cf_clearance import async_cf_retry + + +async def test_cf_challenge(url: str): + # not use cf_clearance, cf challenge is fail + res = requests.get("https://nowsecure.nl") + assert "Just a moment..." in res.text + # get cf_clearance + async with async_playwright() as p: + browser = await p.firefox.launch( + headless=True, + ) + context = await browser.new_context() + page = await context.new_page() + await page.goto(url) + success, cf = await async_cf_retry(page) + if cf: + if success: + cookies = await page.context.cookies() + for cookie in cookies: + if cookie.get("name") == "cf_clearance": + cf_clearance_value = cookie.get("value") + print(cf_clearance_value) + ua = await page.evaluate("() => {return navigator.userAgent}") + assert cf_clearance_value + else: + raise + else: + print("No cloudflare challenges encountered") + + await browser.close() + # use cf_clearance, must be same IP and UA + headers = {"user-agent": ua} + cookies = {"cf_clearance": cf_clearance_value} + res = requests.get("https://nowsecure.nl", headers=headers, cookies=cookies) + assert "Just a moment..." not in res.text + + +if __name__ == "__main__": + import asyncio + + asyncio.run(test_cf_challenge("https://nowsecure.nl"))