How to Scrape Google Reviews: Code and No Code Approaches

const { chromium } = require('playwright');
const fs = require('fs');
async function run() {
  const browser = await chromium.launch({
    headless: false
  });
  /* Custom variables */
  const searchTerm = 'Burj Khalifa';
  // Creating new context and page.
  const context = await browser.newContext();
  const page = await context.newPage();
  // navigating to google.com
  await page.goto('https://www.google.com/');
  // Searching the search term
  await page.getByRole('combobox', { name: 'Search' }).click();
  await page.getByRole('combobox', { name: 'Search' }).type(searchTerm);
  await page.getByRole('combobox', { name: 'Search' }).press('Enter');
  // clicking the review button
  await page.locator('xpath=(//a[@data-async-trigger="reviewDialog"])[1]').click();
  let data = await extractData(page);
  saveData(data);
  // Closing the browser instance
  await context.close();
  await browser.close();
}
/**
 * This function will extract the necessary data.
 * @param {page} page the page object that the data to be scraped.
 * @returns {[object]} The scraped data as object.
 */
async function extractData(page) {
  let dataToSave = [];
  // Necessary selectors.
  const xpathAllReviews = '//div[@jscontroller="fIQYlf"]';
  const xpathMoreButton = "//a[@class='review-more-link']";
  const xpathTitle = "//div[@class='TSUbDb']/a";
  const xpathRating = "//g-review-stars[@class='lTi8oc']/span";
  const xpathReviews = '//span[@jscontroller="MZnM8e"]';
  const allReviews = page.locator(xpathAllReviews);
  const allReviewsCount = await allReviews.count();
  for (var index= 0; index < allReviewsCount ; index++) {
    const element = await allReviews.nth(index);
    // Clicking more button if the review is shortened.
    const moreBtn = element.locator(xpathMoreButton)
    if(await moreBtn.count()>0) {
    try {
        await moreBtn.click();
        await page.waitForTimeout(2500);
    }
    catch {}
    }
    // Scraping necessary data.
    const title = await element.locator(xpathTitle).innerText();
    const rating = await element.locator(xpathRating).getAttribute("aria-label")
    const review = await element.locator(xpathReviews).innerText();
    let rawDataToSave = {
    "author_name": title,
    "rating": rating,
    "review": review
    }
    // Collecting to a list.
    dataToSave.push(rawDataToSave)
  }
  return dataToSave;
}
/**
 * This function used to save the data as json file.
 * @param {[object]} data the data to be written as json file.
 */
function saveData(data) {
  let dataStr = JSON.stringify(data, null, 2)
  fs.writeFile("google_reviews.json", dataStr, 'utf8', function (err) {
    if (err) {
        console.log("An error occurred while writing JSON Object to File.");
        return console.log(err);
    }
    console.log("JSON file has been saved.");
  });
}
run();

import asyncio
import json
from playwright.async_api import Playwright, async_playwright
async def extract_data(page) -> list:
    """
    Extracts the results information from the page
    Args:
        page: Playwright page object
    Returns:
        A list containing details of results as a dictionary. The dictionary
        has title, review count, rating, address of various results
    """
    review_box_xpath = '//div[@jscontroller="fIQYlf"] '
    review_xpath = '//span[@data-expandable-section]'
    secondary_review_xpath = '//span[@class="review-full-text"]'
    author_xpath = '//div[@class="TSUbDb"]'
    rating_xpath = '//g-review-stars/span'
    await page.wait_for_selector(review_box_xpath)
    review_box = page.locator(review_box_xpath)
    data = []
    for review_box_index in range(await review_box.count()):
        result_elem = review_box.nth(review_box_index)
        review = await result_elem.locator(review_xpath).inner_text()
        review = review if review else await result_elem.locator(
            secondary_review_xpath).inner_text()
        author_name = await result_elem.locator(author_xpath).inner_text()
        rating = await result_elem.locator(
            rating_xpath).get_attribute('aria-label')
        rating = rating.strip(', ') if rating else None
        data.append({
            'author_name': author_name,
            'review': review,
            'rating': rating
        })
    return data
async def run(playwright: Playwright) -> None:
    """
    Main function which launches browser instance and performs browser
    interactions
    Args:
        playwright: Playwright instance
    """
    browser = await playwright.chromium.launch(
        headless=False,
        proxy={'server': 'proxy url'}
    )
    context = await browser.new_context()
    # Open new page
    page = await context.new_page()
    # Go to https://www.google.com/
    await page.goto("https://www.google.com/")
    # Type search query
    search_term = "burj khalifa"
    await page.locator("[aria-label=\"Search\"]").type(search_term)
    # Press enter to search in google
    await page.keyboard.press('Enter')
    # wait for review button
    await page.locator(
        '//a[@data-async-trigger="reviewDialog"]').first.wait_for(
        timeout=10000)
    # Click reviews button
    await page.locator('//a[@data-async-trigger="reviewDialog"]').first.click()
    # Initialize the number of pagination required
    pagination_limit = 3
    # Iterate to load reviews for mentioned number of pages
    for page_number in range(pagination_limit):
        await page.locator('//div[@class="review-dialog-list"]').hover()
        await page.mouse.wheel(0, 100000)
        page_number += 1
        await page.wait_for_timeout(2000)
    # Extract all displayed reviews
    data = await extract_data(page)
    # Save all extracted data as a JSON file
    with open('google_reviews.json', 'w') as f:
        json.dump(data, f, indent=2)
    # ---------------------
    await context.close()
    await browser.close()
async def main() -> None:
    async with async_playwright() as playwright:
        await run(playwright)
asyncio.run(main())