Data scraping

In this example, we'll scrape a simple webpage with quotes and authors and save them into an SQLite database.

import requests
from bs4 import BeautifulSoup
from sqlalchemy import create_engine, Column, Integer, String, text
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy_utils import create_database, database_exists

# Define the SQLAlchemy model
Base = declarative_base()

class Quote(Base):
    __tablename__ = 'quotes'

    id = Column(Integer, primary_key=True)
    text = Column(String(512), nullable=False)
    author = Column(String(255), nullable=False)
    tags = Column(String(512), nullable=True)


# Create the engine without specifying the database
engine = create_engine('mysql://username:password@localhost/', echo=True)

# Specify the database name
database_name = 'quotes'

# Check if the 'quotes' database exists, and if not, create it
if not database_exists(engine.url):
    create_database(f"mysql://username:password@localhost/{database_name}")

# Re-create the engine with the 'quotes' database
engine = create_engine(f'mysql://username:password@localhost/{database_name}', echo=True)

# Create the 'quotes' table
Base.metadata.create_all(engine)


# Define a function to scrape and save data
def scrape_and_save_quotes():
    url = 'http://quotes.toscrape.com/page/1/'
    page_number = 1

    Session = sessionmaker(bind=engine)
    session = Session()

    while True:
        response = requests.get(url)
        if response.status_code != 200:
            break

        soup = BeautifulSoup(response.content, 'html.parser')

        for quote_element in soup.select('.quote'):
            text = quote_element.select_one('.text').text
            
            # Check if '.small.author' element exists
            author_element = quote_element.select_one('.small.author')
            author = author_element.text if author_element else 'Unknown Author'
            
            tags = ', '.join(tag.text for tag in quote_element.select('.tag'))

            quote = Quote(text=text, author=author, tags=tags)
            session.add(quote)
        
        session.commit()

        page_number += 1
        next_page = soup.select_one('.next > a')
        if next_page:
            url = f'http://quotes.toscrape.com{next_page["href"]}'
        else:
            break

    session.close()


if __name__ == '__main__':
    scrape_and_save_quotes()

In this code:

  • We send an HTTP GET request to the specified URL using the "requests" library.
  • We parse the HTML content of the page using BeautifulSoup to extract quotes and authors.
  • We create an SQLite database and a table called "quotes" to store the data.
  • We iterate through the scraped data and insert it into the database using SQL INSERT statements.
  • Finally, we commit the changes and close the database connection.

Make sure to replace the URL with the actual website you want to scrape, and you can customize the table and data insertion logic to match your specific needs. This is a basic example, and real-world web scraping tasks may involve more complex data extraction and handling. Additionally, consider checking the website's terms of service and robots.txt file for scraping permissions and limitations.