OUTDATED CODE

[lewitch2025-rgb]

It just doesnt work but I have new code that downloads the scripts

[lewitch2025-rgb]

import os
import time
from urllib.parse import quote, urljoin
from bs4 import BeautifulSoup
import requests

BASE_URL = 'https://www.imsdb.com'
SCRIPTS_DIR = 'scripts'

# --- 1. SETUP: Pretend to be a browser ---
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}

os.makedirs(SCRIPTS_DIR, exist_ok=True)

def clean_script(text):
    text = text.replace('Back to IMSDb', '')
    text = text.replace('Scanned by http://freemoviescripts.com', '')
    text = text.replace('Formatting by http://simplyscripts.home.att.net', '')
    return text.replace(r'\r', '').strip()

def get_script(script_url):
    tail = script_url.split('/')[-1]
    print(f'Fetching: {tail} ...', end=" ")

    try:
        # 1. Fetch the main movie page
        response = requests.get(script_url, headers=HEADERS)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, "html.parser")

        # 2. Find the specific link to the text
        # Look for a link that matches "Read ... Script" structure
        read_link = soup.find('a', href=lambda x: x and x.startswith('/scripts/') and x.endswith('.html'))
        
        if not read_link:
            # Fallback: Find the last link in the center table (classic IMSDb layout)
            center_p = soup.find_all('p', align="center")
            if center_p:
                read_link = center_p[-1].find('a')

        if not read_link:
            print(f'FAILED. (No "Read Script" link found)')
            return None, None

        # 3. Download the actual script text
        final_url = urljoin(BASE_URL, read_link['href'])
        script_response = requests.get(final_url, headers=HEADERS)
        script_soup = BeautifulSoup(script_response.text, "html.parser")
        
        # 4. Extract text (usually in <td class="scrtext">)
        text_area = script_soup.find('td', {'class': "scrtext"})
        
        # Fallback for older pages (<pre>)
        if not text_area:
            text_area = script_soup.find('pre')

        if text_area:
            print("SUCCESS")
            return tail.replace('.html', ''), clean_script(text_area.get_text())
        else:
            print(f'FAILED. (Content hidden or PDF)')
            return None, None

    except Exception as e:
        print(f'ERROR: {e}')
        return None, None

if __name__ == "__main__":
    print("Getting list of scripts...")
    
    try:
        response = requests.get('https://www.imsdb.com/all-scripts.html', headers=HEADERS)
        soup = BeautifulSoup(response.text, "html.parser")

        # Find all script links
        # IMSDb puts them in <td><p><a href="/Movie Scripts/...">
        all_links = soup.find_all('a')
        
        script_links = []
        for link in all_links:
            href = link.get('href')
            if href and href.startswith('/Movie Scripts/') and href.endswith('.html'):
                script_links.append(urljoin(BASE_URL, href))

        print(f"Found {len(script_links)} scripts. Starting download...")

        for link in script_links: 
            title, script_text = get_script(link)
            
            if script_text:
                # Save to file
                safe_title = "".join([c for c in title if c.isalpha() or c.isdigit() or c==' ']).strip()
                filename = os.path.join(SCRIPTS_DIR, safe_title + '.txt')
                
                with open(filename, 'w', encoding='utf-8') as f:
                    f.write(script_text)
            
            # Sleep briefly to be polite to the server
            # time.sleep(0.5) 

    except KeyboardInterrupt:
        print("\nStopped by user.")
    except Exception as e:
        print(f"Critical Error: {e}")