Question

I am trying to bring all items in the webpage-for instance I would like to bring the first

"Hillebrand Boudewynsz. van der Aa (1661 - 1717)" and then all the other 49 in the page MY code is below, I am trying to use selenium and bring the items through xpath or CSS but I am not sure for the right path both options will be welcome this is the required sentence from the code #Finding element object<-remDr$findElement(using="xpath","/html/body/div[2]/div/ul/li[1]/a") #--------------------------------------------------------------------- and the website https://www.vondel.humanities.uva.nl/ecartico/persons/index.php?subtask=browse

rm(list=ls())
library(tidyverse)
#install.packages("robotstxt")
library( robotstxt)
#install.packages("RSelenium")
library(rvest)
library(RSelenium)
library(tidyverse)
#install.packages("netstat")
library(netstat)
library(wdman)
selenium()

# see path
selenium_object<-selenium(retcommand = T,check = F)

#binman::list_versions("chromedriver")

#start the server
remote_driver<-rsDriver(
  
  browser = "chrome",
  
  chromever = "113.0.5672.63",
  verbose = F,
  port = free_port()
)

# create a client object
remDr<-remote_driver$client

#open a browser
remDr$open()

# maximaize window size
remDr$maxWindowSize()

#navigate website
remDr$navigate("https://www.vondel.humanities.uva.nl/ecartico/persons/index.php?subtask=browse")

#Finding element
object<-remDr$findElement(using="xpath","/html/body/div[2]/div/ul/li[1]/a")
#---------------------------------------------------------------------

Answer 1

That should work for you!

import csv
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup

url = "https://www.vondel.humanities.uva.nl/ecartico/persons/index.php?subtask=browse"
all_data = []

# Selenium Configuration
chrome_options = Options()
chrome_options.add_argument("--headless")  # Running in headless mode
driver = webdriver.Chrome(options=chrome_options)
wait = WebDriverWait(driver, 10)

try:
    # Access to page
    driver.get(url)

    # Wait for products to load
    wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "#setwidth > ul > li > a")))

    # Get the HTML content of the page
    html = driver.page_source
finally:
    driver.quit()  # Close the browser even if an exception occurs

# Extracting the necessary data using BeautifulSoup
soup = BeautifulSoup(html,  html.parser )
products = soup.select( #setwidth > ul > li > a )

for title in products:
    title_text = title.get_text(strip=True) if title else ""
    all_data.append([title_text])

# Write data to a CSV file
with open("vondel.csv", "w", newline="", encoding="utf-8") as csv_file:
    writer = csv.writer(csv_file)
    writer.writerow([ Title ])
    writer.writerows(all_data)

友情链接