Scrapping

In this example, we'll scrape a simple webpage with quotes and authors and save them into an MongoDB database.

import requests
from bs4 import BeautifulSoup
from pymongo import MongoClient

# Define the URL to scrape
url = 'http://quotes.toscrape.com/'

# Initialize MongoDB client and connect to the database
client = MongoClient('mongodb://localhost:27017/')  # Change the MongoDB connection URL as needed
db = client['my_quotes_db']  # Change the database name as needed
collection = db['quotes']

def scrape_quotes_and_save_to_mongodb(url):
    page = 1
    while True:
        response = requests.get(url + f'page/{page}/')
        if response.status_code != 200:
            break  # Stop when there are no more pages

        soup = BeautifulSoup(response.text, 'html.parser')
        quotes = soup.select('.quote')

        for quote in quotes:
            text = quote.find(class_='text').get_text()
            
            # Check if the author element exists before calling get_text()
            author_element = quote.find(class_='small')
            author = author_element.get_text() if author_element else ''

            tags = [tag.get_text() for tag in quote.select('.tag')]

            # Create a dictionary to store the quote data
            quote_data = {
                'text': text,
                'author': author,
                'tags': tags
            }

            # Insert the quote data into the MongoDB collection
            collection.insert_one(quote_data)

        page += 1

    print("Scraping and saving to MongoDB complete.")

if __name__ == '__main__':
    scrape_quotes_and_save_to_mongodb(url)

Make sure to adjust the MongoDB connection URL ("mongodb://localhost:27017/"") and the database name ("my_quotes_db") to match your MongoDB setup. This code will start scraping quotes from the website and save them in a collection named 'quotes' in the specified database.