Python: Scrapy

# my_project/middlewares.py
import base64

class ScrapyCatproxyProxyMiddleware:
    def process_request(self, request, spider):
        # По умолчанию — статические настройки прокси
        proxy_user = spider.settings.get('PROXY_USER')
        proxy_pass = spider.settings.get('PROXY_PASS')

        # Позволяет spider’ам переопределять прокси для каждого запроса
        if 'proxy_username_params' in request.meta:
            params = request.meta['proxy_username_params']
            proxy_user = f"{spider.settings.get('PROXY_USER')}{params}"

        # Задайте прокси для запроса
        request.meta['proxy'] = f"http://{spider.settings.get('PROXY_HOST')}:{spider.settings.get('PROXY_PORT')}"

        # Добавьте заголовок Proxy-Authorization для авторизации
        proxy_auth = f"{proxy_user}:{proxy_pass}"
        auth_header = 'Basic ' + base64.b64encode(proxy_auth.encode()).decode()
        request.headers['Proxy-Authorization'] = auth_header

        spider.logger.info(f"Using proxy user: {proxy_user} for {request.url}")

# my_project/settings.py

# --- Catproxy Proxy Credentials ---
PROXY_HOST = "proxy.catproxy.net"
PROXY_PORT = 5555
PROXY_USER = "PROXY_USERNAME" # Without any parameters
PROXY_PASS = "PROXY_PASSWORD"

# --- Enable the Middleware ---
# The number (e.g., 610) should be high enough to run after default middlewares,
# but before any middlewares that might depend on the proxy.
DOWNLOADER_MIDDLEWARES = {
   'my_project.middlewares.ScrapyCatproxyProxyMiddleware': 610,
}

import scrapy
import random

class SteamSpecialsSpider(scrapy.Spider):
    name = "steam_specials"
    start_urls = ["https://store.steampowered.com/specials"]

    def start_requests(self):
        # Create a unique session ID for this job
        session_id = f"steam-special-{random.randint(1000, 9999)}"

        # Define the parameters we want to pass to our middleware
        # We will use the same sticky session for all pages
        proxy_params = f"-country-us-session-{session_id}"

        cookies = {'birthtime': '568022401', 'wants_mature_content': '1', 'steamCountry': 'US|...'}

        for url in self.start_urls:
            yield scrapy.Request(
                url,
                callback=self.parse,
                # Pass our dynamic parameters to the middleware
                meta={'proxy_username_params': proxy_params},
                cookies=cookies,
                headers={
                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36'
                }
            )

    def parse(self, response):
        # Extract data from the current page
        games = response.css('a.salepreviewwidgets_SaleItemBrowserRow_y9MSd')

        if not games:
            self.logger.warning("No games found on page. The CSS selector might be outdated.")
            return

        for game in games:
            yield {
                'title': game.css('div.salepreviewwidgets_StoreSaleWidgetTitle_3jI46::text').get(),
                'original_price': game.css('div.salepreviewwidgets_StoreOriginalPrice_2e_2G::text').get(),
                'discount_price': game.css('div.salepreviewwidgets_StoreSalePriceBox_Wh0L8::text').get(),
            }

        # Find and follow the "Next" page link
        next_page_link = response.css('a.paging_Arrow_2JZe3[href*="specials?page="]:last-of-type').get()
        if next_page_link:
            # Scrapy automatically carries over the `meta` dictionary,
            # so our middleware will continue to use the same session!
            yield response.follow(next_page_link, callback=self.parse)
        else:
            self.logger.info("Last page reached. Finishing crawl.")