How to Scrape Amazon Product Offers and Sellers: Code and No Code Approaches

import asyncio
import json

from playwright.async_api import async_playwright

url = "https://www.amazon.com/dp/B073JYC4XM/ref=olp-opf-redir?aod=1"


async def extract_data(page) -> list:
    """
    Parsing details from the listing page

    Args:
        page: webpage of the browser

    Returns:
        list: details of homes for sale
    """

    # Initializing selectors and xpaths
    div_selector = "[id='aod-offer']"
    name_selector = "[id='aod-asin-title-text']"
    condition_selector = "[id='aod-offer-heading']"
    price_selector = "[class='a-offscreen']"
    seller_selector = "//a[@class='a-size-small a-link-normal' and @role='link']"
    delivery_selector = "//div[@id='aod-offer-shipsFrom']//span[contains(@class,'a-color-base')]"
    seler_rating_xpath = "//i[contains(@class,'a-star-mini')]"
    rating_count_xpath = "[id='seller-rating-count-{iter}']"
    # List to save the details of properties
    amazon_offers_listings = []

    # Waiting for the page to finish loading
    await page.wait_for_selector(div_selector)
    name = await page.locator(name_selector).inner_text() if await page.locator(name_selector).count() else None
    name = clean_data(name)
    # Extracting the elements
    review_cards = page.locator(div_selector)
    cards_count = await review_cards.count()
    for index in range(cards_count):
        # Hovering the element to load the price
        inner_element = review_cards.nth(index=index)
        await inner_element.hover()
        inner_element = review_cards.nth(index=index)
        
        # Extracting necessary data
        condition = await inner_element.locator(condition_selector).inner_text() if await inner_element.locator(condition_selector).count() else None
        price = await inner_element.locator(price_selector).inner_text() if await inner_element.locator(price_selector).count() else None
        seller = await inner_element.locator(seller_selector).inner_text() if await inner_element.locator(seller_selector).count() else None
        delivery = await inner_element.locator(delivery_selector).inner_text() if await inner_element.locator(delivery_selector).count() else None
        rating_data = await inner_element.locator(rating_count_xpath).inner_text() if await inner_element.locator(rating_count_xpath).count() else None
        seller_rate_data = await inner_element.locator(seler_rating_xpath).get_attribute('class') if await inner_element.locator(seler_rating_xpath).count() else None

        # Removing extra spaces and unicode characters
        condition = clean_data(condition)
        price = clean_data(price)
        seller = clean_data(seller)
        delivery = clean_data(delivery)
        rating_data = clean_data(rating_data)
        seller_rate_data = clean_data(seller_rate_data)

        seller_rate = seller_percentage = None
        # extracting data from rating data
        if rating_data:
            data_list = rating_data.split()
            if 'positive' in data_list:
                ind = data_list.index('positive')
                seller_percentage = data_list[ind-1]+ ' ' + 'positive'
            if 'ratings)' in data_list:
                ind = data_list.index('ratings)')
                seller_rating = data_list[ind][1:]+' '+ 'ratings'
        if seller_rate_data:
            data_list = seller_rate_data.split()
            for data in data_list:
                if 'a-star-mini' in data:
                    seller_rate = data
                    break
            seller_rate = seller_rate[12:].replace('-','.')
        data_to_save = {
            "name": name,
            "condition": condition,
            "price": price,
            "seller": seller,
            "delivery": delivery,
            "seller_percentage": seller_percentage,
            "seller_rate":seller_rate
        }

        amazon_offers_listings.append(data_to_save)
        

    save_data(amazon_offers_listings, "Data.json")


async def run(playwright) -> None:
    # Initializing the browser and creating a new page.
    browser = await playwright.firefox.launch(headless=False)
    context = await browser.new_context()
    page = await context.new_page()

    await page.set_viewport_size({"width": 1920, "height": 1080})
    page.set_default_timeout(150000)

    # Navigating to the homepage
    await page.goto(url, wait_until="domcontentloaded")

    await extract_data(page)

    await context.close()
    await browser.close()


def clean_data(data: str) -> str:
    """
    Cleaning data by removing extra white spaces and Unicode characters

    Args:
        data (str): data to be cleaned

    Returns:
        str: cleaned string
    """
    if not data:
        return ""
    cleaned_data = " ".join(data.split()).strip()
    cleaned_data = cleaned_data.encode("ascii", "ignore").decode("ascii")
    return cleaned_data


def save_data(product_page_data: list, filename: str):
    """Converting a list of dictionaries to JSON format

    Args:
        product_page_data (list): details of each product
        filename (str): name of the JSON file
    """
    with open(filename, "w") as outfile:
        json.dump(product_page_data, outfile, indent=4)


async def main() -> None:
    async with async_playwright() as playwright:
        await run(playwright)


if __name__ == "__main__":
    asyncio.run(main())

const { chromium, firefox } = require('playwright');
const fs = require('fs');

const url = "https://www.amazon.com/dp/B073JYC4XM/ref=olp-opf-redir?aod=1";

/**
* Save data as list of dictionaries
as json file
* @param {object} data
*/
function saveData(data) {
    let dataStr = JSON.stringify(data, null, 2)
    fs.writeFile("data.json", dataStr, 'utf8', function (err) {
        if (err) {
            console.log("An error occurred while writing JSON Object to File.");
            return console.log(err);
        }
        console.log("JSON file has been saved.");
    });
}

function cleanData(data) {
    if (!data) {
        return;
    }
    // removing extra spaces and unicode characters
    let cleanedData = data.split(/\s+/).join(" ").trim();
    cleanedData = cleanedData.replace(/[^\x00-\x7F]/g, "");
    return cleanedData;
}

/**
* The data extraction function used to extract
necessary data from the element.
* @param {HtmlElement} innerElement
* @returns
*/
async function extractData(innerElement) {

    async function extractData(data) {
        let count = await data.count();
        if (count) {
            return await data.innerText()
        }
        return null
    };
    // intializing xpath and selectors
    const conditionSelector = "[id='aod-offer-heading']";
    const priceSelector = "[class='a-offscreen']";
    const sellerSelector = "//a[@class='a-size-small a-link-normal' and @role='link']";
    const deliverySelector = "//div[@id='aod-offer-shipsFrom']//span[contains(@class,'a-color-base')]";
    const selerRatingXpath = "//i[contains(@class,'a-star-mini')]";
    const ratingCountXpath = "[id='seller-rating-count-{iter}']";

    let condition = innerElement.locator(conditionSelector);
    condition = await extractData(condition);

    let price = innerElement.locator(priceSelector);
    price = await extractData(price);

    let seller = innerElement.locator(sellerSelector);
    seller = await extractData(seller);

    let delivery = innerElement.locator(deliverySelector);
    delivery = await extractData(delivery);

    let ratingData = innerElement.locator(selerRatingXpath);
    ratingData = await extractData(ratingData);

    let sellerRateData = innerElement.locator(ratingCountXpath);
    sellerRateData = await extractData(sellerRateData);

    // cleaning data
    condition = cleanData(condition)
    price = cleanData(price)
    seller = cleanData(seller)
    delivery = cleanData(delivery)
    ratingData = cleanData(ratingData)
    sellerRateData = cleanData(sellerRateData)

    let sellerPercentage = null;
    let sellerRating = null;

    // Extracting data from rating data
    if (ratingData) {
    const dataList = ratingData.split(' ');
    const positiveIndex = dataList.indexOf('positive');

    if (positiveIndex !== -1) {
        sellerPercentage = dataList[positiveIndex - 1] + ' positive';
    }
    }

    if (sellerRateData) {
    const dataList = sellerRateData.split(' ');
    
    for (const data of dataList) {
        if (data.includes('a-star-mini')) {
        sellerRating = data;
        break;
        }
    }

    if (sellerRating) {
        sellerRating = sellerRating.slice(12).replace('-', '.');
    }
    }


    extractedData = {
        'condition': condition,
        'price': price,
        'seller': seller,
        'delivery': delivery,
        'sellerRating':sellerRating,
        'sellerPercentage':sellerPercentage
    }

    return extractedData
}
/**
* The main function initiate a browser object and handle the navigation.
*/
async function run() {
    // intializing browser and creating new page
    const browser = await firefox.launch({ headless: false });
    const context = await browser.newContext();
    const page = await context.newPage();

    await page.setViewportSize({ width: 1920, height: 1080 });
    page.setDefaultTimeout(120000);
    // intializing xpath and selectors
    const divSelector = "[id='aod-offer']";
    const nameSelector = "[id='aod-asin-title-text']";

    // Navigating to the home page
    await page.goto(url, { waitUntil: 'domcontentloaded' });

    // Wait until the list of properties is loaded
    await page.waitForSelector(divSelector);

    // to store the extracted data
    let data = [];
    await page.waitForLoadState("load");
    await page.waitForTimeout(10);

    let reviewCards = page.locator(divSelector);
    reviewCardsCount = await reviewCards.count()
        // going through each listing element
    for (let index = 0; index < reviewCardsCount; index++) {

        await page.waitForTimeout(2000);
        await page.waitForLoadState("load");
        let innerElement = await reviewCards.nth(index);
        await innerElement.hover();

        innerElement = await reviewCards.nth(index);
        let dataToSave = await extractData(innerElement);
        let name = null;
        // Checking if the element with the given selector exists
        const nameElement = await page.locator(nameSelector).first();
        if (nameElement) {
        // Extract the inner text if the element exists
        name = await nameElement.innerText();
        }
        dataToSave.name = name;
        data.push(dataToSave);
    };
    saveData(data);
    await context.close();
    await browser.close();
};

run();