How To Scrape Amazon.com Products & Reviews [2023]

import requests
from parsel import Selector
from urllib.parse import urljoin

product_urls = []

keyword_list = ['ipad']

for keyword in keyword_list:
    url = f'https://www.amazon.com/s?k={keyword}&page=1'
    try:
        response = requests.get(url)

        if response.status_code == 200:
            sel = Selector(text=response.text)

            ## Extract Product Page URLs
            search_products = sel.css("div.s-result-item[data-component-type=s-search-result]")
            for product in search_products:
                relative_url = product.css("h2>a::attr(href)").get()
                product_url = urljoin('https://www.amazon.com/', relative_url).split("?")[0]
                product_urls.append(product_url) 
            
    except Exception as e:
        print("Error", e)

import requests
from parsel import Selector

product_asins = []

keyword_list = ['ipad']

for keyword in keyword_list:
    url = f'https://www.amazon.com/s?k={keyword}&page=1'
    try:
        response = requests.get(url)

        if response.status_code == 200:
            sel = Selector(text=response.text)

            ## Extract Product ASINS
            search_products = sel.css("div.s-result-item[data-component-type=s-search-result]")
            for product in search_products:
                relative_url = product.css("h2>a::attr(href)").get()
                asin = relative_url.split('/')[3] if len(relative_url.split('/')) >= 4 else None
                product_asins.append(asin) 
            
    except Exception as e:
        print("Error", e)

import requests
from parsel import Selector
from urllib.parse import urljoin

product_urls = []

keyword_list = ['ipad']

for keyword in keyword_list:
    url_list = [f'https://www.amazon.com/s?k={keyword}&page=1']
    for url in url_list:
        try:
            response = requests.get(url)
    
            if response.status_code == 200:
                sel = Selector(text=response.text)

                ## Extract Product Page URLs
                search_products = sel.css("div.s-result-item[data-component-type=s-search-result]")
                for product in search_products:
                    relative_url = product.css("h2>a::attr(href)").get()
                    product_url = urljoin('https://www.amazon.com/', relative_url).split("?")[0]
                    product_urls.append(product_url) 
                
                ## Get All Pages
                if "&page=1" in url:
                    available_pages = sel.xpath(
                        '//a[has-class("s-pagination-item")][not(has-class("s-pagination-separator"))]/text()'
                    ).getall()

                    for page in available_pages:
                        search_url_paginated = f'https://www.amazon.com/s?k={keyword}&page={page}'
                        url_list.append(search_url_paginated)
                
        except Exception as e:
            print("Error", e)

import requests
from parsel import Selector
from urllib.parse import urljoin

keyword_list = ['ipad']
product_overview_data = []

for keyword in keyword_list:
    url_list = [f'https://www.amazon.com/s?k={keyword}&page=1']
    for url in url_list:
        try:
            response = requests.get(url)

            if response.status_code == 200:
                sel = Selector(text=response.text)

                ## Extract Product Page
                search_products = sel.css("div.s-result-item[data-component-type=s-search-result]")
                for product in search_products:
                    relative_url = product.css("h2>a::attr(href)").get()
                    #print(relative_url.split('/'))
                    asin = relative_url.split('/')[3] if len(relative_url.split('/')) >= 4 else None
                    product_url = urljoin('https://www.amazon.com/', relative_url).split("?")[0]
                    product_overview_data.append(
                        {
                            "keyword": keyword,
                            "asin": asin,
                            "url": product_url,
                            "ad": True if "/slredirect/" in product_url else False, 
                            "title": product.css("h2>a>span::text").get(),
                            "price": product.css(".a-price[data-a-size=xl] .a-offscreen::text").get(),
                            "real_price": product.css(".a-price[data-a-size=b] .a-offscreen::text").get(),
                            "rating": (product.css("span[aria-label~=stars]::attr(aria-label)").re(r"(\d+\.*\d*) out") or [None])[0],
                            "rating_count": product.css("span[aria-label~=stars] + span::attr(aria-label)").get(),
                            "thumbnail_url": product.xpath("//img[has-class('s-image')]/@src").get(),
                        }
                    )

                ## Get All Pages
                if "&page=1" in url:
                    available_pages = sel.xpath(
                        '//a[has-class("s-pagination-item")][not(has-class("s-pagination-separator"))]/text()'
                    ).getall()

                    for page in available_pages:
                        search_url_paginated = f'https://www.amazon.com/s?k={keyword}&page={page}'
                        url_list.append(search_url_paginated)
        
        except Exception as e:
            print("Error", e)
            

import re
import requests
from parsel import Selector
from urllib.parse import urljoin

product_urls = [
    'https://www.amazon.com/2021-Apple-10-2-inch-iPad-Wi-Fi/dp/B09G9FPHY6/ref=sr_1_1',
]

product_data_list = []

for product_url in product_urls:
    try:
        response = requests.get(product_url)
    
        if response.status_code == 200:
            sel = Selector(text=response.text)
            image_data = json.loads(re.findall(r"colorImages':.*'initial':\s*(\[.+?\])},\n", response.text)[0])
            variant_data = re.findall(r'dimensionValuesDisplayData"\s*:\s* ({.+?}),\n', response.text)
            feature_bullets = [bullet.strip() for bullet in sel.css("#feature-bullets li ::text").getall()]
            price = sel.css('.a-price span[aria-hidden="true"] ::text').get("")
            if not price:
                price = sel.css('.a-price .a-offscreen ::text').get("")
            product_data_list.append({
                "name": sel.css("#productTitle::text").get("").strip(),
                "price": price,
                "stars": sel.css("i[data-hook=average-star-rating] ::text").get("").strip(),
                "rating_count": sel.css("div[data-hook=total-review-count] ::text").get("").strip(),
                "feature_bullets": feature_bullets,
                "images": image_data,
                "variant_data": variant_data,
            })
    except Exception as e:
            print("Error", e)
    

import requests
from parsel import Selector
from urllib.parse import urljoin

reviews = []

product_review_url_list = ['https://www.amazon.com/product-reviews/B09G9FPHY6/']

for product_review_url in product_review_url_list:

    try:
        response = requests.get(product_review_url)
        
        if response.status_code == 200:    
            sel = Selector(text=response.text)
                        
            ## Parse Product Reviews
            review_elements = sel.css("#cm_cr-review_list div.review")
            for review_element in review_elements:
                reviews.append({
                        "text": "".join(review_element.css("span[data-hook=review-body] ::text").getall()).strip(),
                        "title": review_element.css("*[data-hook=review-title]>span::text").get(),
                        "location_and_date": review_element.css("span[data-hook=review-date] ::text").get(),
                        "verified": bool(review_element.css("span[data-hook=avp-badge] ::text").get()),
                        "rating": review_element.css("*[data-hook*=review-star-rating] ::text").re(r"(\d+\.*\d*) out")[0],
                })
                
    except Exception as e:
        print("Error", e)
    

import requests
from parsel import Selector
from urllib.parse import urljoin

reviews = []

product_review_url_list = ['https://www.amazon.com/product-reviews/B09G9FPHY6/']

for product_review_url in product_review_url_list:
    try:
        response = requests.get(product_review_url)
        
        if response.status_code == 200:    
            sel = Selector(text=response.text)
            
            ## Get Next Page Url
            next_page_relative_url = sel.css(".a-pagination .a-last>a::attr(href)").get()
            if next_page_relative_url is not None:
                next_page = urljoin('https://www.amazon.com/', next_page_relative_url)
                product_review_url_list.append(next_page)
            
            ## Parse Product Reviews
            review_elements = sel.css("#cm_cr-review_list div.review")
            for review_element in review_elements:
                reviews.append({
                        "text": "".join(review_element.css("span[data-hook=review-body] ::text").getall()).strip(),
                        "title": review_element.css("*[data-hook=review-title]>span::text").get(),
                        "location_and_date": review_element.css("span[data-hook=review-date] ::text").get(),
                        "verified": bool(review_element.css("span[data-hook=avp-badge] ::text").get()),
                        "rating": review_element.css("*[data-hook*=review-star-rating] ::text").re(r"(\d+\.*\d*) out")[0],
                })
                
    except Exception as e:
        print("Error", e)
    

SCRAPEOPS_API_KEY = 'YOUR_API_KEY'

def scrapeops_url(url):
    payload = {'api_key': SCRAPEOPS_API_KEY, 'url': url, 'country': 'us'}
    proxy_url = 'https://proxy.scrapeops.io/v1/?' + urlencode(payload)
    return proxy_url


amazon_url = 'https://www.amazon.com/s?k=iPads&page=1'

## Send URL To ScrapeOps Instead of Amazon 
response = requests.get(scrapeops_url(amazon_url))

import requests
from parsel import Selector
from urllib.parse import urlencode, urljoin

API_KEY = 'YOUR_API_KEY'

def scrapeops_url(url):
    payload = {'api_key': API_KEY, 'url': url, 'country': 'us'}
    proxy_url = 'https://proxy.scrapeops.io/v1/?' + urlencode(payload)
    return proxy_url

keyword_list = ['ipad']
product_overview_data = []

for keyword in keyword_list:
    url_list = [f'https://www.amazon.com/s?k={keyword}&page=1']
    for url in url_list:
        try:
            response = requests.get(scrapeops_url(url))

            if response.status_code == 200:
                sel = Selector(text=response.text)

                ## Extract Product Data From Search Page
                search_products = sel.css("div.s-result-item[data-component-type=s-search-result]")
                for product in search_products:
                    relative_url = product.css("h2>a::attr(href)").get()
                    asin = relative_url.split('/')[3] if len(relative_url.split('/')) >= 4 else None
                    product_url = urljoin('https://www.amazon.com/', relative_url).split("?")[0]
                    product_overview_data.append(
                        {
                            "keyword": keyword,
                            "asin": asin,
                            "url": product_url,
                            "ad": True if "/slredirect/" in product_url else False, 
                            "title": product.css("h2>a>span::text").get(),
                            "price": product.css(".a-price[data-a-size=xl] .a-offscreen::text").get(),
                            "real_price": product.css(".a-price[data-a-size=b] .a-offscreen::text").get(),
                            "rating": (product.css("span[aria-label~=stars]::attr(aria-label)").re(r"(\d+\.*\d*) out") or [None])[0],
                            "rating_count": product.css("span[aria-label~=stars] + span::attr(aria-label)").get(),
                            "thumbnail_url": product.xpath("//img[has-class('s-image')]/@src").get(),
                        }
                    )

                ## Get All Pages
                if "&page=1" in url:
                    available_pages = sel.xpath(
                        '//a[has-class("s-pagination-item")][not(has-class("s-pagination-separator"))]/text()'
                    ).getall()

                    for page in available_pages:
                        search_url_paginated = f'https://www.amazon.com/s?k={keyword}&page={page}'
                        url_list.append(search_url_paginated)
        
        except Exception as e:
            print("Error", e)

print(product_overview_data)