Scrapping
In this example, we'll scrape a simple webpage with quotes and authors and save them into an MongoDB database.
import requests
from bs4 import BeautifulSoup
from pymongo import MongoClient
# Define the URL to scrape
url = 'http://quotes.toscrape.com/'
# Initialize MongoDB client and connect to the database
client = MongoClient('mongodb://localhost:27017/') # Change the MongoDB connection URL as needed
db = client['my_quotes_db'] # Change the database name as needed
collection = db['quotes']
def scrape_quotes_and_save_to_mongodb(url):
page = 1
while True:
response = requests.get(url + f'page/{page}/')
if response.status_code != 200:
break # Stop when there are no more pages
soup = BeautifulSoup(response.text, 'html.parser')
quotes = soup.select('.quote')
for quote in quotes:
text = quote.find(class_='text').get_text()
# Check if the author element exists before calling get_text()
author_element = quote.find(class_='small')
author = author_element.get_text() if author_element else ''
tags = [tag.get_text() for tag in quote.select('.tag')]
# Create a dictionary to store the quote data
quote_data = {
'text': text,
'author': author,
'tags': tags
}
# Insert the quote data into the MongoDB collection
collection.insert_one(quote_data)
page += 1
print("Scraping and saving to MongoDB complete.")
if __name__ == '__main__':
scrape_quotes_and_save_to_mongodb(url)
Make sure to adjust the MongoDB connection URL ("mongodb://localhost:27017/"") and the database name ("my_quotes_db") to match your MongoDB setup. This code will start scraping quotes from the website and save them in a collection named 'quotes' in the specified database.