feat: add firefox usage and example (#51)

* feat: add example

Signed-off-by: vvanglro <[email protected]>

* feat: add example

Signed-off-by: vvanglro <[email protected]>

* feat: update every_day_cron_challenge.yml

Signed-off-by: vvanglro <[email protected]>

* feat: update README.md

Signed-off-by: vvanglro <[email protected]>

* feat: update pre-commit

Signed-off-by: vvanglro <[email protected]>

---------

Signed-off-by: vvanglro <[email protected]>

Author: vvanglro

.github/workflows/every_day_cron_challenge.yml

@@ -19,7 +19,9 @@ jobs:
         run: |
           python -m pip install --upgrade pdm
           pdm install
-          pdm run playwright install chromium
+          pdm run playwright install chromium firefox
           sudo apt install xvfb
       - name: Run script
-        run: xvfb-run pdm run python tests/test_cron_challenge.py
+        run: |
+          xvfb-run pdm run python example/test_chromium_cron_challenge.py
+          pdm run python example/test_firefox_cron_challenge.py

.pre-commit-config.yaml

@@ -5,6 +5,7 @@ repos:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
+      - id: check-toml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.0.277
@@ -32,4 +33,4 @@ repos:
         entry: '\bprint\('
         language: pygrep
         types: [ python ]
-        exclude: "tests/test_cron_challenge.py"
+        exclude: "example/test_chromium_cron_challenge.py|example/test_firefox_cron_challenge.py"

README.md

@@ -14,9 +14,11 @@ the cf_clearance, make sure you use the same IP and UA as when you got it.
 
 ## Warning
 
-Please use interface mode, You must add headless=False.
+If you use chromium, Please use interface mode, You must add headless=False.
 If you use it on linux or docker, use XVFB.
 
+If you use firefox you don't need interface mode and XVFB.
+
 Challenge are not always successful. Please try more and handle exceptions.
 
 
@@ -39,22 +41,39 @@ curl http://localhost:8000/challenge -H "Content-Type:application/json" -X POST
 import requests
 
 proxy = "socks5://localhost:7890"
-resp = requests.post("http://localhost:8000/challenge",
-                     json={"proxy": {"server": proxy}, "timeout": 20,
-                           "url": "https://nowsecure.nl"})
+resp = requests.post(
+    "http://localhost:8000/challenge",
+    json={
+        "proxy": {"server": proxy},
+        "timeout": 20,
+        "url": "https://nowsecure.nl",
+        "pure": True,
+        "browser": 2,
+        "cookies": [
+            {
+                "url": "https://www.example.com",
+                "name": "example-cookie",
+                "value": "example-value",
+            }
+        ],
+        "headers": {"example-ua": "example-ua-value"},
+        "exec_js": "() => {return navigator.userAgent}",
+    },
+)
 data = resp.json()
 # In some cases, the cloudflare challenge will not be triggered,
 # so when cf in the return parameter is true, it means that the challenge has been encountered.
 if data.get("success") and data.get("cf"):
     ua = data.get("user_agent")
+    exec_js_resp = data.get("exec_js_resp")
     cf_clearance_value = data.get("cookies").get("cf_clearance")
     # use cf_clearance, must be same IP and UA
     headers = {"user-agent": ua}
     cookies = {"cf_clearance": cf_clearance_value}
-    res = requests.get('https://nowsecure.nl', proxies={
-        "all": proxy
-    }, headers=headers, cookies=cookies)
-    if '<title>Just a moment...</title>' not in res.text:
+    res = requests.get(
+        "https://nowsecure.nl", proxies={"all": proxy}, headers=headers, cookies=cookies
+    )
+    if "<title>Just a moment...</title>" not in res.text:
         print("cf challenge success")
 ```
 
@@ -66,7 +85,7 @@ pip install cf-clearance
 
 ## Usage
 
-Please make sure it is the latest package.
+Please make sure it is the latest package. See [example](https://github.com/vvanglro/cf-clearance/tree/main/example).
 
 ```
 pip install --upgrade cf-clearance
@@ -76,94 +95,7 @@ or
 pip install git+https://github.com/vvanglro/cf-clearance.git@main
 ```
 
-### sync
-
-```python
-from playwright.sync_api import sync_playwright
-from cf_clearance import sync_cf_retry, sync_stealth
-import requests
-
-# not use cf_clearance, cf challenge is fail
-proxies = {
-    "all": "socks5://localhost:7890"
-}
-res = requests.get('https://nowsecure.nl', proxies=proxies)
-if '<title>Just a moment...</title>' in res.text:
-    print("cf challenge fail")
-# get cf_clearance
-with sync_playwright() as p:
-    browser = p.chromium.launch(headless=False, proxy={"server": "socks5://localhost:7890"})
-    page = browser.new_page()
-    sync_stealth(page, pure=True)
-    page.goto('https://nowsecure.nl')
-    res, cf = sync_cf_retry(page)
-    if cf:
-        if res:
-            cookies = page.context.cookies()
-            for cookie in cookies:
-                if cookie.get('name') == 'cf_clearance':
-                    cf_clearance_value = cookie.get('value')
-                    print(cf_clearance_value)
-            ua = page.evaluate('() => {return navigator.userAgent}')
-            print(ua)
-        else:
-            print("cf challenge fail")
-    else:
-        print("No cloudflare challenges encountered")
-    browser.close()
-# use cf_clearance, must be same IP and UA
-headers = {"user-agent": ua}
-cookies = {"cf_clearance": cf_clearance_value}
-res = requests.get('https://nowsecure.nl', proxies=proxies, headers=headers, cookies=cookies)
-if '<title>Just a moment...</title>' not in res.text:
-    print("cf challenge success")
-```
-
-### async
-
-```python
-import asyncio
-from playwright.async_api import async_playwright
-from cf_clearance import async_cf_retry, async_stealth
-import requests
-
-
-async def main():
-    # not use cf_clearance, cf challenge is fail
-    proxies = {
-        "all": "socks5://localhost:7890"
-    }
-    res = requests.get('https://nowsecure.nl', proxies=proxies)
-    if '<title>Just a moment...</title>' in res.text:
-        print("cf challenge fail")
-    # get cf_clearance
-    async with async_playwright() as p:
-        browser = await p.chromium.launch(headless=False, proxy={"server": "socks5://localhost:7890"})
-        page = await browser.new_page()
-        await async_stealth(page, pure=True)
-        await page.goto('https://nowsecure.nl')
-        res, cf = await async_cf_retry(page)
-        if cf:
-            if res:
-                cookies = await page.context.cookies()
-                for cookie in cookies:
-                    if cookie.get('name') == 'cf_clearance':
-                        cf_clearance_value = cookie.get('value')
-                        print(cf_clearance_value)
-                ua = await page.evaluate('() => {return navigator.userAgent}')
-                print(ua)
-            else:
-                print("cf challenge fail")
-        else:
-            print("No cloudflare challenges encountered")
-        await browser.close()
-    # use cf_clearance, must be same IP and UA
-    headers = {"user-agent": ua}
-    cookies = {"cf_clearance": cf_clearance_value}
-    res = requests.get('https://nowsecure.nl', proxies=proxies, headers=headers, cookies=cookies)
-    if '<title>Just a moment...</title>' not in res.text:
-        print("cf challenge success")
-
-
-asyncio.get_event_loop().run_until_complete(main())
+## Install Playwright Depends
+```shell
+playwright install chromium firefox
 ```

example/__init__.py

@@ -1,9 +1,14 @@
+import requests
 from playwright.async_api import async_playwright
 
 from cf_clearance import async_cf_retry, async_stealth
 
 
 async def test_cf_challenge(url: str):
+    # not use cf_clearance, cf challenge is fail
+    res = requests.get("https://nowsecure.nl")
+    assert "<title>Just a moment...</title>" in res.text
+    # get cf_clearance
     async with async_playwright() as p:
         browser = await p.chromium.launch(
             headless=False,
@@ -12,19 +17,27 @@ async def test_cf_challenge(url: str):
         page = await context.new_page()
         await async_stealth(page, pure=True)
         await page.goto(url)
-        res, cf = await async_cf_retry(page)
+        success, cf = await async_cf_retry(page)
         if cf:
-            if res:
+            if success:
                 cookies = await page.context.cookies()
                 for cookie in cookies:
                     if cookie.get("name") == "cf_clearance":
                         cf_clearance_value = cookie.get("value")
+                print(cf_clearance_value)
+                ua = await page.evaluate("() => {return navigator.userAgent}")
                 assert cf_clearance_value
             else:
                 raise
         else:
             print("No cloudflare challenges encountered")
+
         await browser.close()
+    # use cf_clearance, must be same IP and UA
+    headers = {"user-agent": ua}
+    cookies = {"cf_clearance": cf_clearance_value}
+    res = requests.get("https://nowsecure.nl", headers=headers, cookies=cookies)
+    assert "<title>Just a moment...</title>" not in res.text
 
 
 if __name__ == "__main__":

tests/test_cron_challenge.py renamed to example/test_chromium_cron_challenge.py

@@ -0,0 +1,45 @@
+import requests
+from playwright.async_api import async_playwright
+
+from cf_clearance import async_cf_retry
+
+
+async def test_cf_challenge(url: str):
+    # not use cf_clearance, cf challenge is fail
+    res = requests.get("https://nowsecure.nl")
+    assert "<title>Just a moment...</title>" in res.text
+    # get cf_clearance
+    async with async_playwright() as p:
+        browser = await p.firefox.launch(
+            headless=True,
+        )
+        context = await browser.new_context()
+        page = await context.new_page()
+        await page.goto(url)
+        success, cf = await async_cf_retry(page)
+        if cf:
+            if success:
+                cookies = await page.context.cookies()
+                for cookie in cookies:
+                    if cookie.get("name") == "cf_clearance":
+                        cf_clearance_value = cookie.get("value")
+                print(cf_clearance_value)
+                ua = await page.evaluate("() => {return navigator.userAgent}")
+                assert cf_clearance_value
+            else:
+                raise
+        else:
+            print("No cloudflare challenges encountered")
+
+        await browser.close()
+    # use cf_clearance, must be same IP and UA
+    headers = {"user-agent": ua}
+    cookies = {"cf_clearance": cf_clearance_value}
+    res = requests.get("https://nowsecure.nl", headers=headers, cookies=cookies)
+    assert "<title>Just a moment...</title>" not in res.text
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(test_cf_challenge("https://nowsecure.nl"))