Source Code: Web Scraping Per-market and After Hour Screeners Data For Stock Analysis with Python

import datetime
import requests
from bs4 import BeautifulSoup
import pandas as pd

# url = "https://www.marketwatch.com/tools/screener/after-hours?mod=side_nav"
url = "https://www.marketwatch.com/tools/screener/premarket"
headers= {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0'}

response = requests.get(url, headers=headers)
soup = BeautifulSoup(response.content, 'html.parser')

element_tables = soup.find_all('div', 'element element--table table--fixed screener-table')

current_datetime = datetime.datetime.now().strftime('%m-%d-%y %H_%M_%S')
xlsxwriter = pd.ExcelWriter('Permarket Screener ({0}).xlsx'.format(current_datetime))

dfs = {}

for element_table in element_tables:
    df_table = pd.read_html(str(element_table))[0]
    header = element_table.find('div', 'element__head')
    screener_name = header.h2.text
    table_updated = header.div.small.text
    print(f'{screener_name} table_updated: {table_updated}')
    dfs[screener_name] = df_table
    dfs[screener_name].to_excel(xlsxwriter, sheet_name=screener_name, index=False)

xlsxwriter.save()