#!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import logging import time from datetime import datetime from decimal import Decimal, InvalidOperation import mysql.connector as mysql from selenium import webdriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.common.exceptions import ( InvalidSessionIdException, WebDriverException, TimeoutException ) # ========== CONFIG ========== DB_HOST = "192.168.0.41" DB_USER = "mawdz09" DB_PASS = "asaxPI1324" DB_NAME = "marketdb" CHROMEDRIVER_PATH = "/usr/bin/chromedriver" CHROMIUM_BIN_CANDIDATES = ( "/usr/bin/chromium", "/usr/bin/chromium-browser", ) LOG_LEVEL = os.environ.get("SCRAPER_LOG_LEVEL", "INFO").upper() MAX_RETRIES = 2 # retries per URL RESTART_WAIT = 3 # seconds to wait before restarting Chrome # ============================ URLS = { "Nasdaq Composite": "https://www.investing.com/indices/nasdaq-composite", "FTSE 100": "https://www.investing.com/indices/uk-100", "DAX": "https://www.investing.com/indices/germany-30", "Euro Stoxx 50": "https://www.investing.com/indices/eu-stoxx50", "S&P 500 (SPX)": "https://uk.investing.com/indices/us-spx-500", "Nikkei 225 (N225)": "https://uk.investing.com/indices/japan-ni225", "CBOE Volatility Index": "https://www.investing.com/indices/volatility-s-p-500", } def setup_logging(): logging.basicConfig( level=getattr(logging, LOG_LEVEL, logging.INFO), format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S", ) def get_driver(): chrome_opts = Options() chrome_opts.add_argument("--headless=new") chrome_opts.add_argument("--no-sandbox") chrome_opts.add_argument("--disable-gpu") chrome_opts.add_argument("--disable-dev-shm-usage") chrome_opts.add_argument("--window-size=1200,900") chrome_opts.add_argument("--disable-extensions") chrome_opts.add_argument("--disable-background-networking") chrome_opts.add_argument("--memory-pressure-off") chrome_opts.add_argument( "--user-agent=Mozilla/5.0 (X11; Linux armv8l) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/124 Safari/537.36" ) for candidate in CHROMIUM_BIN_CANDIDATES: if os.path.exists(candidate): chrome_opts.binary_location = candidate break service = Service(CHROMEDRIVER_PATH) return webdriver.Chrome(service=service, options=chrome_opts) def is_session_dead(e: Exception) -> bool: """Returns True if Chrome has crashed and needs a full restart.""" return isinstance(e, (InvalidSessionIdException, WebDriverException)) and ( "invalid session id" in str(e).lower() or "session deleted" in str(e).lower() or "chrome not reachable" in str(e).lower() or "failed to decode response" in str(e).lower() ) def parse_number(txt: str) -> Decimal: if txt is None: raise InvalidOperation("empty value") cleaned = ( txt.replace(",", "") .replace("%", "") .replace("(", "") .replace(")", "") .replace("+", "") .strip() ) if cleaned in ("", "—", "-", "NaN"): raise InvalidOperation(f"bad numeric: {txt!r}") return Decimal(cleaned) def ensure_table(conn): sql = """ CREATE TABLE IF NOT EXISTS index_quotes ( id INT AUTO_INCREMENT PRIMARY KEY, ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, index_name VARCHAR(64) NOT NULL, price DECIMAL(12,2) NOT NULL, change_abs DECIMAL(12,2) NOT NULL, change_pct DECIMAL(7,2) NOT NULL, KEY idx_ts (ts), KEY idx_name (index_name) ) ENGINE=InnoDB; """ with conn.cursor() as cur: cur.execute(sql) conn.commit() def scrape_one(driver, name: str, url: str): logging.info("Loading %s", url) driver.get(url) wait = WebDriverWait(driver, 20) price_sel = (By.CSS_SELECTOR, 'span[data-test="instrument-price-last"]') change_sel = (By.CSS_SELECTOR, 'span[data-test="instrument-price-change"]') pct_sel = (By.CSS_SELECTOR, 'span[data-test="instrument-price-change-percent"]') price_el = wait.until(EC.visibility_of_element_located(price_sel)) change_el = wait.until(EC.visibility_of_element_located(change_sel)) pct_el = wait.until(EC.visibility_of_element_located(pct_sel)) price = parse_number(price_el.text) chg = parse_number(change_el.text) pct = parse_number(pct_el.text) return {"index_name": name, "price": price, "change_abs": chg, "change_pct": pct} def insert_row(conn, row): sql = """ INSERT INTO index_quotes (index_name, price, change_abs, change_pct) VALUES (%s, %s, %s, %s) """ vals = (row["index_name"], str(row["price"]), str(row["change_abs"]), str(row["change_pct"])) with conn.cursor() as cur: cur.execute(sql, vals) conn.commit() def main(): setup_logging() logging.info("Scrape run started") conn = mysql.connect(host=DB_HOST, user=DB_USER, password=DB_PASS, database=DB_NAME) try: ensure_table(conn) driver = get_driver() ok = fail = restarts = 0 try: for name, url in URLS.items(): success = False for attempt in range(MAX_RETRIES + 1): try: row = scrape_one(driver, name, url) insert_row(conn, row) logging.info( "Inserted: %-24s | Price %s | Δ %s (%s%%)", row["index_name"], row["price"], row["change_abs"], row["change_pct"], ) ok += 1 success = True break except Exception as e: if is_session_dead(e): # Chrome crashed — restart it logging.warning( "Chrome session dead on %s (attempt %d) — restarting browser", name, attempt + 1 ) try: driver.quit() except Exception: pass time.sleep(RESTART_WAIT) driver = get_driver() restarts += 1 elif isinstance(e, TimeoutException) and attempt < MAX_RETRIES: logging.warning( "Timeout on %s (attempt %d) — retrying", name, attempt + 1 ) time.sleep(2) else: logging.exception("Failed for %s: %s", name, e) break if not success: fail += 1 finally: try: driver.quit() except Exception: pass logging.info( "Scrape run finished — %d ok, %d failed, %d browser restart(s)", ok, fail, restarts ) finally: conn.close() if __name__ == "__main__": main()