Repository: omkarcloud/amazon-scraper
Branch: master
Commit: ff9fdae6fd72
Files: 20
Total size: 55.1 KB
Directory structure:
gitextract_4xhu7oyx/
├── .gitignore
├── .gitpod.yml
├── Dockerfile
├── LICENSE
├── README.md
├── SECURITY.md
├── advanced.md
├── cache/
│ ├── get_product/
│ │ └── a77e2b6d09fe1229ea74e803a284d424.json
│ └── search/
│ └── 97e2eb5cf27ecd24880993346e62705b.json
├── docker-compose.yaml
├── local_storage.json
├── main.py
├── profiles.json
├── requirements.txt
└── src/
├── __init__.py
├── amazon_scraper.py
├── search.py
├── utils.py
├── write_output.py
└── write_output_utils.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Botasaurus stuff:
build/
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
tasks/
profiles/
output/
create_cache.py
error_logs/
gpt/
.history/
================================================
FILE: .gitpod.yml
================================================
image: chetan1111/gitpod-botasaurus:3
tasks:
- init: pip install -r requirements.txt
================================================
FILE: Dockerfile
================================================
FROM chetan1111/botasaurus:latest
ENV PYTHONUNBUFFERED=1
COPY requirements.txt .
RUN python -m pip install -r requirements.txt
RUN mkdir app
WORKDIR /app
COPY . /app
CMD ["python", "main.py"]
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2023 Chetan Jain
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
# Amazon Scraper API
Scrape Amazon products, prices, reviews, and categories from 24 marketplaces via a simple REST API. 100 free requests/month.
## Key Features
- Search Amazon products, get product details, browse by category, and fetch top reviews — all via 1 API.
- 100 free queries per month. No credit card required.
Here's a sample response for a **product search results page**:
```json
{
"title": "Apple iPhone 15, 128GB, Black - Unlocked (Renewed)",
"price": 403.0,
"rating": 4.1,
"reviews": 2769,
"asin": "B0CMPMY9ZZ",
"link": "https://www.amazon.com/dp/B0CMPMY9ZZ",
"image_url": "https://m.media-amazon.com/images/I/51PtFHUPjBL._AC_UY654_FMwebp_QL65_.jpg",
"currency": "USD",
"is_best_seller": false,
"is_amazon_choice": false,
"is_prime": false,
"sales_volume": "2K+ bought in past month"
}
```
## Get API Key
Create an account at [omkar.cloud](https://www.omkar.cloud/auth/sign-up?redirect=/api-key) to get your API key.
It takes just 2 minutes to sign up. You get 100 free requests every month for detailed Amazon data.
This is a well built product, and your search for the best Amazon Scraper API ends right here.
## Quick Start
```bash
curl -X GET "https://amazon-scraper-api.omkar.cloud/amazon/search?query=iPhone%2016" \
-H "API-Key: YOUR_API_KEY"
```
```json
{
"results": [
{
"title": "Apple iPhone 15, 128GB, Black - Unlocked (Renewed)",
"price": 403.0,
"rating": 4.1,
"reviews": 2769,
"asin": "B0CMPMY9ZZ",
"link": "https://www.amazon.com/dp/B0CMPMY9ZZ",
"currency": "USD",
"is_best_seller": false,
"is_amazon_choice": false,
"is_prime": false,
"sales_volume": "2K+ bought in past month"
}
]
}
```
## Quick Start (Python)
```bash
pip install requests
```
```python
import requests
# Search for products
response = requests.get(
"https://amazon-scraper-api.omkar.cloud/amazon/search",
params={"query": "iPhone 16", "country_code": "US"},
headers={"API-Key": "YOUR_API_KEY"}
)
print(response.json())
```
## API Reference
### Product Search
```
GET https://amazon-scraper-api.omkar.cloud/amazon/search
```
#### Parameters
| Parameter | Required | Default | Description |
|-----------|----------|---------|-------------|
| `query` | Yes | — | Search query. Keywords or ASIN. |
| `page` | No | `1` | Page number. |
| `country_code` | No | `US` | Amazon marketplace code. |
| `sort_by` | No | `relevance` | `relevance`, `lowest_price`, `highest_price`, `reviews`, `newest`, `best_sellers` |
#### Example
```python
import requests
response = requests.get(
"https://amazon-scraper-api.omkar.cloud/amazon/search",
params={"query": "iPhone 16", "country_code": "US"},
headers={"API-Key": "YOUR_API_KEY"}
)
print(response.json())
```
#### Response
Sample Response (click to expand)
```json
{
"results": [
{
"title": "Apple iPhone 15, 128GB, Black - Unlocked (Renewed)",
"price": 403.0,
"original_price": null,
"rating": 4.1,
"reviews": 2769,
"asin": "B0CMPMY9ZZ",
"link": "https://www.amazon.com/dp/B0CMPMY9ZZ",
"image_url": "https://m.media-amazon.com/images/I/51PtFHUPjBL._AC_UY654_FMwebp_QL65_.jpg",
"currency": "USD",
"is_best_seller": false,
"is_amazon_choice": false,
"is_prime": false,
"delivery_info": "FREE delivery Fri, Feb 13",
"number_of_offers": 62,
"lowest_offer_price": 359.99,
"has_variations": true,
"sales_volume": "2K+ bought in past month",
"is_climate_friendly": false
}
]
}
```
---
### Products by Category
```
GET https://amazon-scraper-api.omkar.cloud/amazon/products/category
```
#### Parameters
| Parameter | Required | Default | Description |
|-----------|----------|---------|-------------|
| `category_id` | Yes | — | Amazon category ID (from URL `node=` param). |
| `page` | No | `1` | Page number. |
| `country_code` | No | `US` | Amazon marketplace code. |
| `sort_by` | No | `relevance` | `relevance`, `lowest_price`, `highest_price`, `reviews`, `newest`, `best_sellers` |
#### Example
```python
import requests
response = requests.get(
"https://amazon-scraper-api.omkar.cloud/amazon/products/category",
params={"category_id": "16225007011", "country_code": "US"},
headers={"API-Key": "YOUR_API_KEY"}
)
print(response.json())
```
#### Response
Sample Response (click to expand)
```json
{
"results": [
{
"title": "Apple iPhone 15, 128GB, Black - Unlocked (Renewed)",
"price": 403.0,
"original_price": null,
"rating": 4.1,
"reviews": 2769,
"asin": "B0CMPMY9ZZ",
"link": "https://www.amazon.com/dp/B0CMPMY9ZZ",
"image_url": "https://m.media-amazon.com/images/I/51PtFHUPjBL._AC_UY654_FMwebp_QL65_.jpg",
"currency": "USD",
"is_best_seller": false,
"is_amazon_choice": false,
"is_prime": false,
"delivery_info": "FREE delivery Fri, Feb 13",
"number_of_offers": 62,
"lowest_offer_price": 359.99,
"has_variations": true,
"sales_volume": "2K+ bought in past month",
"is_climate_friendly": false
}
]
}
```
---
### Product Details
```
GET https://amazon-scraper-api.omkar.cloud/amazon/product-details
```
#### Parameters
| Parameter | Required | Default | Description |
|-----------|----------|---------|-------------|
| `asin` | Yes | — | Amazon ASIN (e.g., `B0FWD726XF`). Also accepts product URLs. |
| `country_code` | No | `US` | Amazon marketplace code. |
#### Example
```python
import requests
response = requests.get(
"https://amazon-scraper-api.omkar.cloud/amazon/product-details",
params={"asin": "B0FWD726XF", "country_code": "US"},
headers={"API-Key": "YOUR_API_KEY"}
)
print(response.json())
```
#### Response Fields
Returns 50+ fields including price, rating, description, key features, technical specs, all images/videos, rating distribution, product variants, category hierarchy, brand info, frequently bought together, and top reviews.
Sample Response (click to expand)
```json
{
"asin": "B0FWD726XF",
"product_name": "Apple 2025 MacBook Pro Laptop with M5 chip with 10‑core CPU and GPU: Built for Apple Intelligence, 14.2-inch Liquid Retina XDR Display, 24GB Unified Memory, 1TB SSD Storage, Space Black",
"link": "https://www.amazon.com/dp/B0FWD726XF",
"slug": "Apple-2025-MacBook-Laptop-10%E2%80%91core",
"parent_asin": "B0DLRGKGTV",
"landing_asin": "B0FWD726XF",
"brand_info": "Visit the Apple Store",
"brand_url": "https://www.amazon.com/stores/Apple/page/77D9E1F7-0337-4282-9DB6-B6B8FB2DC98D?lp_asin=B0FWD726XF&ref_=ast_bln",
"brand_urls": [
"https://www.amazon.com/stores/Apple/page/77D9E1F7-0337-4282-9DB6-B6B8FB2DC98D?lp_asin=B0FWD726XF&ref_=ast_bln"
],
"current_price": 1849.0,
"original_price": 1999.0,
"unit_price": null,
"unit_count": null,
"currency": "USD",
"min_order_quantity": null,
"country": "US",
"availability": "In Stock",
"condition": "Buy new:",
"number_of_offers": 6,
"delivery_info": "FREE delivery Monday, February 16 Or Prime members get FREE delivery Tomorrow, February 12. Join Prime",
"estimated_delivery_date": "Monday, February 16",
"rating": 4.7,
"reviews": 1167,
"detailed_rating": {
"1": 3,
"2": 0,
"3": 3,
"4": 5,
"5": 89
},
"customer_feedback_summary": null,
"top_reviews": [
{
"review_id": "R1H9MVROAP2XKG",
"product_asin": "B0FWD726XF",
"review_title": "Sleek, powerful, and a photographers dream. 11/10",
"review_text": "Okay so I actually had to return this item with amazon because they messed up delivery...",
"review_link": "https://www.amazon.com/gp/customer-reviews/R1H9MVROAP2XKG",
"rating": 5,
"review_date": "Reviewed in the United States on December 1, 2025",
"is_verified_purchase": true,
"helpful_votes": 32,
"reviewer_name": "Catherine Mason",
"reviewer_id": "AF77JTSDDJJG6FXTYJAQOG4B4QFA",
"reviewer_url": "https://www.amazon.com/gp/profile/amzn1.account.AF77JTSDDJJG6FXTYJAQOG4B4QFA",
"reviewer_avatar": "https://m.media-amazon.com/images/S/amazon-avatars-global/default.png",
"review_images": [],
"review_video": null,
"reviewed_variant": {
"Style": "Apple M5 chip",
"Capacity": "16GB Unified Memory, 512GB SSD Storage",
"Color": "Space Black",
"Set": "Without AppleCare+"
},
"is_vine_review": false
},
...
],
"is_bestseller": false,
"is_amazon_choice": true,
"is_prime": true,
"is_climate_friendly": true,
"sales_volume": "1K+ bought in past month",
"main_image_url": "https://m.media-amazon.com/images/I/6112T6g2P-L._AC_SL1500_.jpg",
"additional_image_urls": [
"https://m.media-amazon.com/images/I/6112T6g2P-L._AC_SL1500_.jpg",
"https://m.media-amazon.com/images/I/61SBJYmPyFL._AC_SL1500_.jpg",
"https://m.media-amazon.com/images/I/81uUolI6viL._AC_SL1500_.jpg",
"https://m.media-amazon.com/images/I/616eNe+uHRL._AC_.jpg",
"https://m.media-amazon.com/images/I/81cfIW5+p7L._AC_SL1500_.jpg",
"https://m.media-amazon.com/images/I/71c2cMlWe5L._AC_SL1500_.jpg",
"https://m.media-amazon.com/images/I/61EQmd9L4kL._AC_SL1500_.jpg"
],
"product_videos": [
{
"id": "amzn1.vse.video.0882c1076e654eab890412db954d0944",
"title": "MacBook Pro M5 Chip",
"url": "https://m.media-amazon.com/images/S/vse-vms-transcoding-artifact-us-east-1-prod/a60f6616-cd5d-4a84-a49a-b0ca7c22effb/default.jobtemplate.hls.m3u8",
"height": 1080,
"width": 1920,
"thumbnail": "https://m.media-amazon.com/images/I/9103IPwwY2L._SX35_SY46._CR0,0,35,46_BG85,85,85_BR-120_PKdp-play-icon-overlay__.png",
"product_id": "B0FWD726XF",
"parent_id": "B0DLRGKGTV"
},
{
"id": "amzn1.vse.video.007d9729d15a4dc7b533d61249583a76",
"title": "AppleCare+ for Mac",
"url": "https://m.media-amazon.com/images/S/vse-vms-transcoding-artifact-us-east-1-prod/3f731513-0ce4-4f42-ba76-e97deb21bcc7/default.jobtemplate.hls.m3u8",
"height": 1080,
"width": 1920,
"thumbnail": "https://m.media-amazon.com/images/I/71N+KnY1REL._SX35_SY46._CR0,0,35,46_BG85,85,85_BR-120_PKdp-play-icon-overlay__.png",
"product_id": "B0FWD726XF",
"parent_id": "B0DLRGKGTV"
}
],
"user_videos": [],
"video_thumbnail": "https://m.media-amazon.com/images/I/9103IPwwY2L.SX522_.png",
"has_video": true,
"key_features": [
"SUPERCHARGED BY M5 — The 14-inch MacBook Pro with M5 brings next-generation speed and powerful on-device AI to personal, professional, and creative tasks. Featuring all-day battery life and a breathtaking Liquid Retina XDR display with up to 1600 nits peak brightness, it's pro in every way.*",
"HAPPILY EVER FASTER — Along with its faster CPU and unified memory, M5 features a more powerful GPU with a Neural Accelerator built into each core, delivering faster AI performance. So you can blaze through demanding workloads at mind-bending speeds.",
"BUILT FOR APPLE INTELLIGENCE — Apple Intelligence is the personal intelligence system that helps you write, express yourself, and get things done effortlessly. With groundbreaking privacy protections, it gives you peace of mind that no one else can access your data — not even Apple.*",
"ALL-DAY BATTERY LIFE — MacBook Pro delivers the same exceptional performance whether it's running on battery or plugged in.",
"APPS FLY WITH APPLE SILICON — All your favorites, including Microsoft 365 and Adobe Creative Cloud, run lightning fast in macOS.*",
"IF YOU LOVE IPHONE, YOU'LL LOVE MAC — Mac works like magic with your other Apple devices. View and control what's on your iPhone from your Mac with iPhone Mirroring.* Copy something on iPhone and paste it on Mac. Send texts with Messages or use your Mac to make and answer FaceTime calls.*",
"BRILLIANT PRO DISPLAY — The 14.2-inch Liquid Retina XDR display features 1600 nits peak brightness, up to 1000 nits sustained brightness, and 1,000,000:1 contrast.*",
"ADVANCED CAMERA AND AUDIO — Stay perfectly framed and sound great with a 12MP Center Stage camera, three studio-quality mics, and six speakers with Spatial Audio and support for Dolby Atmos.",
"CONNECT IT ALL — This MacBook Pro features three Thunderbolt 4 ports and a MagSafe 3 charging port, SDXC card slot, HDMI port, and headphone jack. And it supports up to two external displays.",
"* LEGAL DISCLAIMERS — This is a summary of the main product features. See below to learn more."
],
"full_description": null,
"technical_details": {
"Product Dimensions": "12.31 x 8.71 x 0.61 inches",
"Item Weight": "3.41 pounds",
"Manufacturer": "Apple",
"ASIN": "B0FWD726XF",
"Item model number": "MDE34LL/A",
"Batteries": "1 Lithium Ion batteries required. (included)",
"Date First Available": "October 14, 2025"
},
"product_details": {
"Brand": "Apple",
"Model Name": "MacBook Pro",
"Screen Size": "14.2 Inches",
"Color": "Space Black",
"Hard Disk Size": "1 TB",
"CPU Model": "Unknown",
"Ram Memory Installed Size": "24 GB",
"Operating System": "Mac OS",
"Special Feature": "Backlit Keyboard, Fingerprint Reader",
"Graphics Card Description": "Integrated"
},
"main_category": {
"id": "aps",
"name": "All Departments"
},
"category_hierarchy": [],
"variation_dimensions": [
"style",
"size",
"color",
"configuration"
],
"variants": {
"style": [
{
"asin": "B0FWD726XF",
"value": "Apple M5 chip",
"is_available": true
},
{
"asin": "B0DLHBGBW3",
"value": "Apple M4 Pro chip",
"is_available": true
},
{
"asin": "B0DLHCXF81",
"value": "Apple M4 Max chip",
"is_available": false
}
],
"size": [
{
"asin": "B0FWD623D1",
"value": "16GB Unified Memory, 1TB SSD Storage",
"is_available": true
},
{
"asin": "B0FWD6SKL6",
"value": "16GB Unified Memory, 512GB SSD Storage",
"is_available": true
},
{
"asin": "B0FWD726XF",
"value": "24GB Unified Memory, 1TB SSD Storage",
"is_available": true
},
{
"asin": "B0DLHY2BJ6",
"value": "24GB Unified Memory, 512GB SSD Storage",
"is_available": false
},
{
"asin": "B0DLHCXF81",
"value": "36GB Unified Memory, 1TB SSD Storage",
"is_available": false
}
],
"color": [
{
"asin": "B0FWD726XF",
"value": "Space Black",
"photo": "https://m.media-amazon.com/images/I/01B-RyYQGML.jpg",
"is_available": true
},
{
"asin": "B0FWD7QF6M",
"value": "Silver",
"photo": "https://m.media-amazon.com/images/I/01xh++YLubL.jpg",
"is_available": true
}
],
"configuration": [
{
"asin": "B0FWD726XF",
"value": "Without AppleCare+",
"is_available": true
},
{
"asin": "B0FWLNSJ1M",
"value": "With AppleCare+ (3 Years)",
"is_available": true
}
]
},
"all_variants": {
"B0FWLNSJ1M": {
"style": "Apple M5 chip",
"size": "24GB Unified Memory, 1TB SSD Storage",
"color": "Space Black",
"configuration": "With AppleCare+ (3 Years)"
},
"B0FWD5MR3L": {
"style": "Apple M5 chip",
"size": "16GB Unified Memory, 512GB SSD Storage",
"color": "Silver",
"configuration": "Without AppleCare+"
},
"B0DLHDJH98": {
"style": "Apple M4 Pro chip",
"size": "24GB Unified Memory, 512GB SSD Storage",
"color": "Silver",
"configuration": "Without AppleCare+"
},
"B0FWKR7V14": {
"style": "Apple M5 chip",
"size": "24GB Unified Memory, 1TB SSD Storage",
"color": "Silver",
"configuration": "With AppleCare+ (3 Years)"
},
"B0DM6YG983": {
"style": "Apple M4 Pro chip",
"size": "24GB Unified Memory, 1TB SSD Storage",
"color": "Space Black",
"configuration": "With AppleCare+ (3 Years)"
},
"B0FWKTY1NM": {
"style": "Apple M5 chip",
"size": "16GB Unified Memory, 1TB SSD Storage",
"color": "Space Black",
"configuration": "With AppleCare+ (3 Years)"
},
"B0DM6ZP92H": {
"style": "Apple M4 Pro chip",
"size": "24GB Unified Memory, 512GB SSD Storage",
"color": "Space Black",
"configuration": "With AppleCare+ (3 Years)"
},
"B0DLHBGBW3": {
"style": "Apple M4 Pro chip",
"size": "24GB Unified Memory, 1TB SSD Storage",
"color": "Space Black",
"configuration": "Without AppleCare+"
},
"B0DLHY2BJ6": {
"style": "Apple M4 Pro chip",
"size": "24GB Unified Memory, 512GB SSD Storage",
"color": "Space Black",
"configuration": "Without AppleCare+"
},
"B0FWD6SKL6": {
"style": "Apple M5 chip",
"size": "16GB Unified Memory, 512GB SSD Storage",
"color": "Space Black",
"configuration": "Without AppleCare+"
},
"B0FWKWYYKS": {
"style": "Apple M5 chip",
"size": "16GB Unified Memory, 512GB SSD Storage",
"color": "Silver",
"configuration": "With AppleCare+ (3 Years)"
},
"B0FWD623D1": {
"style": "Apple M5 chip",
"size": "16GB Unified Memory, 1TB SSD Storage",
"color": "Space Black",
"configuration": "Without AppleCare+"
},
"B0FWD7QF6M": {
"style": "Apple M5 chip",
"size": "24GB Unified Memory, 1TB SSD Storage",
"color": "Silver",
"configuration": "Without AppleCare+"
},
"B0FWKT6VRG": {
"style": "Apple M5 chip",
"size": "16GB Unified Memory, 512GB SSD Storage",
"color": "Space Black",
"configuration": "With AppleCare+ (3 Years)"
},
"B0DLHGN9P2": {
"style": "Apple M4 Pro chip",
"size": "24GB Unified Memory, 1TB SSD Storage",
"color": "Silver",
"configuration": "Without AppleCare+"
},
"B0FWKV16DL": {
"style": "Apple M5 chip",
"size": "16GB Unified Memory, 1TB SSD Storage",
"color": "Silver",
"configuration": "With AppleCare+ (3 Years)"
},
"B0FWD726XF": {
"style": "Apple M5 chip",
"size": "24GB Unified Memory, 1TB SSD Storage",
"color": "Space Black",
"configuration": "Without AppleCare+"
},
"B0FWD8WBSW": {
"style": "Apple M5 chip",
"size": "16GB Unified Memory, 1TB SSD Storage",
"color": "Silver",
"configuration": "Without AppleCare+"
},
"B0DLHCXF81": {
"style": "Apple M4 Max chip",
"size": "36GB Unified Memory, 1TB SSD Storage",
"color": "Space Black",
"configuration": "Without AppleCare+"
},
"B0DM733GXY": {
"style": "Apple M4 Pro chip",
"size": "24GB Unified Memory, 512GB SSD Storage",
"color": "Silver",
"configuration": "With AppleCare+ (3 Years)"
}
},
"has_aplus_content": false,
"aplus_images": [],
"has_brand_story": false,
"frequently_bought_together": []
}
```
---
### Top Product Reviews
```
GET https://amazon-scraper-api.omkar.cloud/amazon/product-reviews/top
```
#### Parameters
| Parameter | Required | Default | Description |
|-----------|----------|---------|-------------|
| `asin` | Yes | — | Amazon ASIN. |
| `country_code` | No | `US` | Amazon marketplace code. |
#### Example
```python
import requests
response = requests.get(
"https://amazon-scraper-api.omkar.cloud/amazon/product-reviews/top",
params={"asin": "B0FWD726XF", "country_code": "US"},
headers={"API-Key": "YOUR_API_KEY"}
)
print(response.json())
```
#### Response
Sample Response (click to expand)
```json
{
"results": [
{
"review_id": "R1H9MVROAP2XKG",
"product_asin": "B0FWD726XF",
"review_title": "Sleek, powerful, and a photographers dream. 11/10",
"review_text": "The battery life is good for how powerful it is. Overall super happy with this laptop. Would buy again every time.",
"review_link": "https://www.amazon.com/gp/customer-reviews/R1H9MVROAP2XKG",
"rating": 5,
"review_date": "Reviewed in the United States on December 1, 2025",
"is_verified_purchase": true,
"helpful_votes": 32,
"reviewer_name": "Catherine Mason",
"reviewer_id": "AF77JTSDDJJG6FXTYJAQOG4B4QFA",
"reviewer_url": "https://www.amazon.com/gp/profile/amzn1.account.AF77JTSDDJJG6FXTYJAQOG4B4QFA",
"reviewer_avatar": "https://m.media-amazon.com/images/S/amazon-avatars-global/default.png",
"review_images": [],
"review_video": null,
"reviewed_variant": {
"Style": "Apple M5 chip",
"Capacity": "16GB Unified Memory, 512GB SSD Storage",
"Color": "Space Black",
"Set": "Without AppleCare+"
},
"is_vine_review": false
}
]
}
```
### Supported Amazon Marketplaces
`US`, `AU`, `BR`, `CA`, `CN`, `FR`, `DE`, `IN`, `IT`, `MX`, `NL`, `SG`, `ES`, `TR`, `AE`, `GB`, `JP`, `SA`, `PL`, `SE`, `BE`, `EG`, `ZA`, `IE`
## Error Handling
```python
response = requests.get(
"https://amazon-scraper-api.omkar.cloud/amazon/search",
params={"query": "iPhone 16"},
headers={"API-Key": "YOUR_API_KEY"}
)
if response.status_code == 200:
data = response.json()
elif response.status_code == 401:
# Invalid API key
pass
elif response.status_code == 429:
# Rate limit exceeded
pass
```
## FAQs
### What data does the API return?
**Product Search** returns per product:
- Title, price, original price, currency
- Star rating, review count, sales volume
- ASIN, product URL, image URL
- Best Seller, Amazon's Choice, and Prime badges
- Delivery info, number of offers, lowest offer price
**Product Details** returns 50+ fields including:
- Full description, key features, technical specs
- All product images and videos
- Rating distribution breakdown
- Product variations (size, color, etc.)
- Category hierarchy, brand info
- Frequently bought together items
- Customer feedback summary and top reviews
**Top Reviews** returns per review:
- Review title, text, rating, and date
- Verified purchase status, helpful vote count
- Reviewer name, avatar, and profile link
- Review images and videos
All in structured JSON. Ready to use in your app.
### How accurate is the data?
Data is pulled from Amazon in real time. Every API call fetches live data — not cached or stale results. Prices, availability, ratings, and reviews reflect what's on Amazon right now.
### What's the difference between Product Search and Products by Category?
**Product Search** finds products matching a keyword or ASIN — like typing into the Amazon search bar.
**Products by Category** returns all products within a specific Amazon category (e.g., "Electronics" or "Books > Science Fiction"). Pass a `category_id` from the Amazon URL, get a paginated list of every product in that category.
Use Search when you know what you're looking for. Use Category when you want to explore or monitor an entire product segment.
## Rate Limits
| Plan | Price | Requests/Month |
|------|-------|----------------|
| Free | $0 | 100 |
| Starter | $16 | 15,000 |
| Grow | $48 | 75,000 |
| Scale | $148 | 300,000 |
## Questions? We have answers.
Reach out anytime. We will solve your query within 1 working day.
[](https://api.whatsapp.com/send?phone=918178804274&text=I%20have%20a%20question%20about%20the%20Amazon%20Scraper%20API.)
[](mailto:happy.to.help@omkar.cloud?subject=Amazon%20Scraper%20API%20Question)
================================================
FILE: SECURITY.md
================================================
# Security Policy for the Amazon Scraper Project
## Disclaimer
By using Amazon Scraper, you agree to comply with all applicable local and international laws related to data scraping, copyright, and privacy. The developers of Amazon Scraper will not be held liable for any misuse of this software. It is the user's sole responsibility to ensure adherence to all relevant laws regarding data scraping, copyright, and privacy, and to use Amazon Scraper in an ethical and legal manner, in line with both local and international regulations.
## 1. Reporting a Vulnerability
We take the privacy and security of the Amazon Scraper Project very seriously. If you have discovered a security vulnerability or have concerns about the project, we appreciate your assistance in responsibly disclosing it to us.
To report a security issue or express a concern, please email Chetan Jain at [chetan@omkar.cloud](mailto:chetan@omkar.cloud). We will promptly respond to your concerns.
## 2. Use at Your Own Risk
This project is provided for ethical and legal purposes only. It must be used in compliance with all relevant local and international laws and is not intended for unauthorized or illegal use.
## 3. Contact
For questions regarding this security policy or the security of the Amazon Scraper Project, please contact [chetan@omkar.cloud](mailto:chetan@omkar.cloud).
The information in this `SECURITY.md` is provided "as is," without any kind of warranty.
================================================
FILE: advanced.md
================================================
## 🤔 Advanced Questions
### ❓ I don't have Python installed. How can I run the scraper?
You can easily run the scraper in Gitpod, a browser-based development environment. Set it up in just 5 minutes by following these steps:
1. Visit [this link](https://gitpod.io/#https://github.com/omkarcloud/amazon-scraper) and sign up using your GitHub account.
2. Once signed up, open it in Gitpod.
3. In the terminal, run the following command to start scraping:
```bash
python main.py
```
4. Once the scraper has finished running, download the data from the `output` folder.
### ❓ Need More Help or Have Additional Questions?
For further help, contact us on WhatsApp. We'll be happy to help you out.
[](https://api.whatsapp.com/send?phone=918295042963&text=Hi,%20I%20would%20like%20to%20learn%20more%20about%20your%20products.)
## Love It? [Star It! ⭐](https://github.com/omkarcloud/amazon-scraper/stargazers)
## Made with ❤️ using [Botasaurus Web Scraping Framework](https://github.com/omkarcloud/botasaurus)
================================================
FILE: cache/get_product/a77e2b6d09fe1229ea74e803a284d424.json
================================================
{
"data": {
"title": "Late 2019 Apple MacBook Pro with 2.6GHz Intel Core i7 (16-Inch, 16GB RAM, 512GB Storage) - Silver (Renewed)",
"description": "Late 2019 Apple MacBook Pro with 2.6GHz Intel Core i7 (16-Inch, 16GB RAM, 512GB Storage) - Silver (Renewed)",
"price": 875.0,
"seller": "JemJem",
"features": [
"Ninth-generation 6-Core Intel Core i7 Processor",
"Stunning 16-inch Retina Display with True Tone technology",
"Touch Bar and Touch ID",
"Amd Radeon Pro 5300M Graphics with GDDR6 memory",
"Ultrafast SSD",
"Intel UHD Graphics 630",
"Six-speaker system with force-cancelling woofers",
"Four Thunderbolt 3 (USB-C) ports",
"Up to 11 hours of battery life",
"802.11AC Wi-Fi"
],
"rating": 4.3,
"reviews": 111,
"images": [
"https://m.media-amazon.com/images/I/619L9jf3-rL._AC_SL1500_.jpg",
"https://m.media-amazon.com/images/I/71u7ZoxwLkL._AC_SL1500_.jpg",
"https://m.media-amazon.com/images/I/919EXGgh2aL._AC_SL1500_.jpg",
"https://m.media-amazon.com/images/I/91yAgiEZqjL._AC_SL1500_.jpg",
"https://m.media-amazon.com/images/I/71XSEMx26HL._AC_SL1500_.jpg",
"https://m.media-amazon.com/images/I/51k92mFqIcL._AC_SL1500_.jpg"
],
"alternatives": [],
"delivery_status": "FREE delivery Thursday, January 4. ",
"questions_answered_count": 21,
"videos_count": 0
},
"error": null
}
================================================
FILE: cache/search/97e2eb5cf27ecd24880993346e62705b.json
================================================
{
"data": {
"count": 18,
"results": [
{
"title": "2020 MacBook Air Laptop M1 Chip, 13\u201d Retina Display, 8GB RAM, 256GB SSD Storage, Backlit Keyboard, FaceTime HD Camera, Touch ID. Works with iPhone/iPad; Gold",
"link": "https://www.amazon.com/gp/bestsellers/pc/13896615011/ref=sr_bs_0_13896615011_1",
"currency": "$",
"price": 949,
"rating": 4,
"reviews": 19039,
"featured_image": "https://m.media-amazon.com/images/I/71vFKBpKakL._AC_UY218_.jpg",
"is_prime": true,
"is_best_seller": true,
"is_amazon_choice": false
},
{
"title": "2023 MacBook Air Laptop with M2 chip: 15.3-inch Liquid Retina Display, 16GB Unified Memory, 256GB SSD Storage, 1080p FaceTime HD Camera, Touch ID. Works with iPhone/iPad; Midnight",
"link": "https://www.amazon.com/sspa/click?ie=UTF8&spc=MToyNTI5ODI1NDYwMzcwMDA6MTcwNDA5MjUyOTpzcF9hdGY6MzAwMDY1ODg5ODQyMzAyOjowOjo&url=%2FApple-2023-MacBook-Laptop-chip%2Fdp%2FB0CDJL36W4%2Fref%3Dsr_1_2_sspa%3Fkeywords%3Dmacbook%26qid%3D1704092529%26sr%3D8-2-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9hdGY%26psc%3D1",
"currency": "$",
"price": 1499,
"rating": 4,
"reviews": 1300,
"featured_image": "https://m.media-amazon.com/images/I/81EHBhjC-+L._AC_UY218_.jpg",
"is_prime": true,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "2023 MacBook Air Laptop with M2 chip: 15.3-inch Liquid Retina Display, 16GB Unified Memory, 256GB SSD Storage, 1080p FaceTime HD Camera, Touch ID. Works with iPhone/iPad; Midnight",
"link": "https://www.amazon.com/Apple-2023-MacBook-Laptop-chip/dp/B0CDJL36W4/ref=sr_1_3?keywords=macbook&qid=1704092529&sr=8-3",
"currency": "$",
"price": 1499,
"rating": 4,
"reviews": 1300,
"featured_image": "https://m.media-amazon.com/images/I/81EHBhjC-+L._AC_UY218_.jpg",
"is_prime": true,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "2020 MacBook Air Laptop M1 Chip, 13\u201d Retina Display, 8GB RAM, 256GB SSD Storage, Backlit Keyboard, FaceTime HD Camera, Touch ID. Works with iPhone/iPad; Silver",
"link": "https://www.amazon.com/gp/bestsellers/pc/13896615011/ref=sr_bs_3_13896615011_1",
"currency": "$",
"price": 949,
"rating": 4,
"reviews": 19039,
"featured_image": "https://m.media-amazon.com/images/I/71TPda7cwUL._AC_UY218_.jpg",
"is_prime": true,
"is_best_seller": true,
"is_amazon_choice": false
},
{
"title": "2023 MacBook Pro Laptop M3 chip with 8\u2011core CPU, 10\u2011core GPU: 14.2-inch Liquid Retina XDR Display, 8GB Unified Memory, 1TB SSD Storage. Works with iPhone/iPad; Silver",
"link": "https://www.amazon.com/Apple-MacBook-Laptop-8%E2%80%91core-10%E2%80%91core/dp/B0CM5LZWJ1/ref=sr_1_5?keywords=macbook&qid=1704092529&sr=8-5",
"currency": "$",
"price": 1748,
"rating": 4,
"reviews": 29,
"featured_image": "https://m.media-amazon.com/images/I/61HUaSA6c0L._AC_UY218_.jpg",
"is_prime": true,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "2022 MacBook Pro Laptop with M2 chip: 13-inch Retina Display, 8GB RAM, 512GB \u200b\u200b\u200b\u200b\u200b\u200b\u200bSSD \u200b\u200b\u200b\u200b\u200b\u200b\u200bStorage, Touch Bar, Backlit Keyboard, FaceTime HD Camera. Works with iPhone and iPad; Silver",
"link": "https://www.amazon.com/2022-Apple-MacBook-Laptop-chip/dp/B0B3C8XMXM/ref=sr_1_6?keywords=macbook&qid=1704092529&sr=8-6",
"currency": "$",
"price": 1430,
"rating": 4,
"reviews": 1206,
"featured_image": "https://m.media-amazon.com/images/I/61bX2AoGj2L._AC_UY218_.jpg",
"is_prime": true,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "2022 MacBook Air Laptop with M2 chip: 13.6-inch Liquid Retina Display, 8GB RAM, 256GB SSD Storage, Backlit Keyboard, 1080p FaceTime HD Camera. Works with iPhone and iPad; Silver",
"link": "https://www.amazon.com/2022-Apple-MacBook-Laptop-chip/dp/B0B3CDZLTB/ref=sr_1_7?keywords=macbook&qid=1704092529&sr=8-7",
"currency": "$",
"price": 1049,
"rating": 4,
"reviews": 2343,
"featured_image": "https://m.media-amazon.com/images/I/71eXNIDUGjL._AC_UY218_.jpg",
"is_prime": false,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "Apple MacBook Air with Intel Core i5, 1.6GHz, (13-inch, 4GB,128GB SSD) - Silver (Renewed)",
"link": "https://www.amazon.com/Apple-MacBook-MJVE2LL-13-inch-Refurbished/dp/B013HD3INW/ref=sr_1_8?keywords=macbook&qid=1704092529&sr=8-8",
"currency": "$",
"price": 249,
"rating": 4,
"reviews": 1973,
"featured_image": "https://m.media-amazon.com/images/I/6157eU56s2L._AC_UY218_.jpg",
"is_prime": false,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "13.3\" MacBook Air with Retina Display - MWT92LL/A",
"link": "https://www.amazon.com/Apple-13-3-MacBook-Retina-Display/dp/B09KW8NRHZ/ref=sr_1_9?keywords=macbook&qid=1704092529&sr=8-9",
"currency": "$",
"price": 1199,
"rating": 0,
"reviews": 0,
"featured_image": "https://m.media-amazon.com/images/I/31WRiCGHeZL._AC_UY218_.jpg",
"is_prime": true,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "Late 2019 Apple MacBook Pro with 2.6GHz Intel Core i7 (16-Inch, 16GB RAM, 512GB Storage) - Silver (Renewed)",
"link": "https://www.amazon.com/Apple-MacBook-16-Inch-Storage-2-6GHz/dp/B08CZT64VP/ref=sr_1_10?keywords=macbook&qid=1704092529&sr=8-10",
"currency": "$",
"price": 875,
"rating": 4,
"reviews": 111,
"featured_image": "https://m.media-amazon.com/images/I/619L9jf3-rL._AC_UY218_.jpg",
"is_prime": false,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "2023 MacBook Pro Laptop M3 chip with 8\u2011core CPU, 10\u2011core GPU: 14.2-inch Liquid Retina XDR Display, 16GB Unified Memory, 512GB SSD Storage, Space Gray (Z1C80001D)",
"link": "https://www.amazon.com/Apple-MacBook-Laptop-8%E2%80%91core-10%E2%80%91core/dp/B0CN9RFFVT/ref=sr_1_11?keywords=macbook&qid=1704092529&sr=8-11",
"currency": "$",
"price": 1759,
"rating": 0,
"reviews": 0,
"featured_image": "https://m.media-amazon.com/images/I/61lsexTCOhL._AC_UY218_.jpg",
"is_prime": false,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "2018 Apple MacBook Pro with 2.3GHz Intel Core i5 (13-inch, 8GB RAM, 256GB SSD Storage) (QWERTY English) Space Gray (Renewed)",
"link": "https://www.amazon.com/Apple-MacBook-13-inch-Storage-English/dp/B07P6C16BC/ref=sr_1_12?keywords=macbook&qid=1704092529&sr=8-12",
"currency": "$",
"price": 472,
"rating": 4,
"reviews": 687,
"featured_image": "https://m.media-amazon.com/images/I/51NUmIQeUzL._AC_UY218_.jpg",
"is_prime": false,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "MacBook Air 13.3\" with Retina Display, M1 Chip with 8-Core CPU and 7-Core GPU, 16GB Memory, 512GB SSD, Space Gray, Late 2020",
"link": "https://www.amazon.com/Apple-MacBook-Air-Retina-Display/dp/B08VF621Z4/ref=sr_1_13?keywords=macbook&qid=1704092529&sr=8-13",
"currency": "$",
"price": 1362,
"rating": 4,
"reviews": 157,
"featured_image": "https://m.media-amazon.com/images/I/51KhexN7YkL._AC_UY218_.jpg",
"is_prime": false,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "2022 13\" MacBook Air M2, 16GB RAM, 1TB Storage - Midnight (Z160000B6)",
"link": "https://www.amazon.com/Apple-2022-MacBook-16GB-Storage/dp/B0BBXRK4WH/ref=sr_1_14?keywords=macbook&qid=1704092529&sr=8-14",
"currency": "$",
"price": 1699,
"rating": 4,
"reviews": 10,
"featured_image": "https://m.media-amazon.com/images/I/719C6bJv8jL._AC_UY218_.jpg",
"is_prime": true,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "Apple MacBook Air MD711LL/A 11.6-inch Laptop - Intel Core i5 1.3GHz - 4GB RAM - 128GB SSD (Renewed)",
"link": "https://www.amazon.com/Apple-MacBook-MD711LL-11-6-Inch-Refurbished/dp/B01550MWHI/ref=sr_1_15?keywords=macbook&qid=1704092529&sr=8-15",
"currency": "$",
"price": 299,
"rating": 4,
"reviews": 2032,
"featured_image": "https://m.media-amazon.com/images/I/71Z3zEs2vYL._AC_UY218_.jpg",
"is_prime": false,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "2020 MacBook Pro M1 Chip (13-inch, 8GB RAM, 256GB SSD Storage) - Space Gray",
"link": "https://www.amazon.com/Apple-MacBook-13-inch-256GB-Storage/dp/B08N5N6RSS/ref=sr_1_16?keywords=macbook&qid=1704092529&sr=8-16",
"currency": "",
"price": 0,
"rating": 4,
"reviews": 7029,
"featured_image": "https://m.media-amazon.com/images/I/71an9eiBxpL._AC_UY218_.jpg",
"is_prime": false,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "Apple MacBook Air MD761LL/A 13.3-Inch Laptop (OLD VERSION) (Renewed)",
"link": "https://www.amazon.com/Apple-MacBook-MD761LL-13-3-Inch-Refurbished/dp/B00M4LWW0Y/ref=sr_1_17?keywords=macbook&qid=1704092529&sr=8-17",
"currency": "$",
"price": 289,
"rating": 3,
"reviews": 91,
"featured_image": "https://m.media-amazon.com/images/I/11rXkb9f7DL._AC_UY218_.jpg",
"is_prime": false,
"is_best_seller": false,
"is_amazon_choice": false
},
{
"title": "2023 MacBook Pro Laptop M3 Pro chip with 12\u2011core CPU, 18\u2011core GPU: 14.2-inch Liquid Retina XDR Display, 18GB Unified Memory, 1TB SSD Storage. Works with iPhone/iPad; Space Black",
"link": "https://www.amazon.com/Apple-MacBook-Laptop-12%E2%80%91core-18%E2%80%91core/dp/B0CM5KK44S/ref=sr_1_18?keywords=macbook&qid=1704092529&sr=8-18",
"currency": "$",
"price": 2369,
"rating": 4,
"reviews": 39,
"featured_image": "https://m.media-amazon.com/images/I/61RJn0ofUsL._AC_UY218_.jpg",
"is_prime": false,
"is_best_seller": false,
"is_amazon_choice": false
}
]
},
"error": null
}
================================================
FILE: docker-compose.yaml
================================================
version: "3"
services:
bot-1:
restart: "no"
shm_size: 800m
build:
dockerfile: Dockerfile
context: .
volumes:
- ./output:/app/output
- ./tasks:/app/tasks
- ./profiles:/app/profiles
- ./profiles.json:/app/profiles.json
- ./local_storage.json:/app/local_storage.json
================================================
FILE: local_storage.json
================================================
{
"count": 47,
"credits_used": 55
}
================================================
FILE: main.py
================================================
from src import Amazon
queries = [
"Macbook",
]
Amazon.search(queries)
================================================
FILE: profiles.json
================================================
{}
================================================
FILE: requirements.txt
================================================
botasaurus
casefy
================================================
FILE: src/__init__.py
================================================
from .amazon_scraper import Amazon
================================================
FILE: src/amazon_scraper.py
================================================
from typing import List,Optional, Union, Dict
from botasaurus import bt
from .write_output import write_output
from .search import FAILED_DUE_TO_CREDITS_EXHAUSTED, FAILED_DUE_TO_NO_KEY,FAILED_DUE_TO_NOT_SUBSCRIBED, FAILED_DUE_TO_UNKNOWN_ERROR, get_product, search
def clean_data(social_details):
success, credits_exhausted, not_subscribed, unknown_error, no_key = [], [], [], [], []
for detail in social_details:
if detail.get("error") is None:
success.append(detail)
elif detail["error"] == FAILED_DUE_TO_CREDITS_EXHAUSTED:
credits_exhausted.append(detail)
elif detail["error"] == FAILED_DUE_TO_NOT_SUBSCRIBED:
not_subscribed.append(detail)
elif detail["error"] == FAILED_DUE_TO_UNKNOWN_ERROR:
unknown_error.append(detail)
elif detail["error"] == FAILED_DUE_TO_NO_KEY:
no_key.append(detail)
return success, credits_exhausted, not_subscribed, unknown_error, no_key
def print_data_errors(credits_exhausted, not_subscribed, unknown_error, no_key):
if credits_exhausted:
name = "queries" if len(credits_exhausted) > 1 else "query"
print(f"Could not get data for {len(credits_exhausted)} {name} due to credit exhaustion. Please consider upgrading your plan by visiting https://rapidapi.com/Chetan11dev/api/amazon-product-scraper8/pricing to continue scraping data.")
if not_subscribed:
name = "queries" if len(not_subscribed) > 1 else "query"
print(f"Could not get data for {len(not_subscribed)} {name} as you are not subscribed to Amazon Scraper API. Please subscribe to a free plan by visiting https://rapidapi.com/Chetan11dev/api/amazon-product-scraper8/pricing")
if unknown_error:
name = "queries" if len(unknown_error) > 1 else "query"
print(f"Could not get data for {len(unknown_error)} {name} due to Unknown Error.")
if no_key:
name = "queries" if len(no_key) > 1 else "query"
print(f"Could not get data for {len(no_key)} {name} as you are not subscribed to Amazon Scraper API. Please subscribe to a free plan by visiting https://rapidapi.com/Chetan11dev/api/amazon-product-scraper8/pricing")
class Amazon:
@staticmethod
def search(query: Union[str, List[str]], key: Optional[str] =None, use_cache: bool = True) -> Dict:
"""
Function to scrape data from Amazon.
:param use_cache: Boolean indicating whether to use cached data.
:return: List of dictionaries with the scraped data.
"""
cache = use_cache
if isinstance(query, str):
query = [query]
max = None
query = [{"query":query_query, "max": max} for query_query in query]
result = []
for item in query:
# TODO: max fixes
data = item
metadata = {"key": key}
result_item = search(data, cache=cache, metadata=metadata)
success, credits_exhausted, not_subscribed, unknown_error, no_key = clean_data([result_item])
print_data_errors(credits_exhausted, not_subscribed, unknown_error, no_key)
if success:
data = result_item.get('data')
if not data:
data = {}
result_item = data.get('results', [])
result.extend(result_item)
write_output(item['query'], result_item,None)
if result:
# bt.write_json(result, "result")
write_output('_all',result, None, lambda x:x)
search.close()
return result
@staticmethod
def get_products(asin: Union[str, List[str]], key: Optional[str] =None, use_cache: bool = True) -> Dict:
"""
Function to scrape data from Amazon.
:param use_cache: Boolean indicating whether to use cached data.
:return: List of dictionaries with the scraped data.
"""
cache = use_cache
if isinstance(asin, str):
asin = [asin]
asin = [{"asin":query_query} for query_query in asin]
result = []
for item in asin:
# TODO: max fixes
data = item
metadata = {"key": key}
result_item = get_product(data, cache=cache, metadata=metadata)
success, credits_exhausted, not_subscribed, unknown_error, no_key = clean_data([result_item])
print_data_errors(credits_exhausted, not_subscribed, unknown_error, no_key)
if success:
data = result_item.get('data')
if not data:
data = {}
result_item = data
result.append(result_item)
# write_output(item['query'], result_item,None)
if result:
# bt.write_json(result, "result")
write_output('products',result, None, lambda x:x)
search.close()
return result
================================================
FILE: src/search.py
================================================
from botasaurus import bt
from botasaurus.cache import DontCache
from botasaurus import cl
from time import sleep
from botasaurus import *
from .utils import default_request_options
import requests
FAILED_DUE_TO_CREDITS_EXHAUSTED = "FAILED_DUE_TO_CREDITS_EXHAUSTED"
FAILED_DUE_TO_NOT_SUBSCRIBED = "FAILED_DUE_TO_NOT_SUBSCRIBED"
FAILED_DUE_TO_NO_KEY = "FAILED_DUE_TO_NO_KEY"
FAILED_DUE_TO_UNKNOWN_ERROR = "FAILED_DUE_TO_UNKNOWN_ERROR"
def update_credits():
credits_used = bt.LocalStorage.get_item("credits_used", 0)
bt.LocalStorage.set_item("credits_used", credits_used + 1)
def do_request(data, retry_count=3):
params = data["params"]
link = params["link"]
key = data["key"]
# print(params)
# print("link", link)
if retry_count == 0:
print(f"Failed to get data, after 3 retries")
return {
"data": None,
"error":FAILED_DUE_TO_UNKNOWN_ERROR,
}
headers = {
"X-RapidAPI-Key": key,
"X-RapidAPI-Host": "amazon-product-scraper8.p.rapidapi.com"
}
response = requests.get(link, headers=headers)
response_data = response.json()
if response.status_code == 200 or response.status_code == 404:
message = response_data.get("message", "")
if "API doesn't exists" in message:
return {
"data": None,
"error":FAILED_DUE_TO_UNKNOWN_ERROR
}
update_credits()
# print(response_data)
# bt.write_json(response_data, "response.json")
if response.status_code == 404:
print(f"No data found")
return {
"data": response_data,
"error": None
}
return {
"data": response_data,
"error": None
}
else:
message = response_data.get("message", "")
if "exceeded the MONTHLY quota" in message:
return {
"data": None,
"error":FAILED_DUE_TO_CREDITS_EXHAUSTED
}
elif "exceeded the rate limit per second for your plan" in message or "many requests" in message:
sleep(2)
return do_request(data, retry_count - 1)
elif "You are not subscribed to this API." in message:
return {
"data": None,
"error": FAILED_DUE_TO_NOT_SUBSCRIBED
}
print(f"Error: {response.status_code}", response_data)
return {
"data": None,
"error":FAILED_DUE_TO_UNKNOWN_ERROR,
}
@request(**default_request_options)
def search(_, data, metadata):
if not metadata.get('key'):
return DontCache({
"data": None,
"error":FAILED_DUE_TO_NO_KEY
})
max_items = data['max']
url = "https://amazon-product-scraper8.p.rapidapi.com/search/"
qp = {"query": data['query']}
params = {**qp, 'link':cl.join_link(url, query_params=qp)}
request_data = {**metadata, "params": params}
result = do_request(request_data)
initial_results = cl.select(result, 'data', 'results', default=[])
if not cl.select(result, 'error'):
more_results = cl.select(result, 'data', 'results', default=[])
print(f"Got {len(more_results)} more results")
while cl.select(result, 'data', 'next') and (max_items is None or len(initial_results) < max_items):
next = cl.select(result, 'data', 'next')
params = {**qp, 'link':next}
request_data = {**metadata, "params": params}
result = do_request(request_data)
if result.get('error'):
break
more_results = cl.select(result, 'data', 'results', default=[])
print(f"Got {len(more_results)} more results")
initial_results.extend(more_results)
if cl.select(result, 'error'):
return DontCache(result)
else:
if max_items is not None:
initial_results = initial_results[:max_items]
result['data']['results'] = initial_results
return result
@request(**default_request_options)
def get_product(_, data, metadata):
if not metadata.get('key'):
return DontCache({
"data": None,
"error":FAILED_DUE_TO_NO_KEY
})
url = "https://amazon-product-scraper8.p.rapidapi.com/product/"
qp = {"asin": data['asin']}
params = {**qp, 'link':cl.join_link(url, query_params=qp)}
request_data = {**metadata, "params": params}
result = do_request(request_data)
return result
================================================
FILE: src/utils.py
================================================
default_browser_options = {
"block_images": True,
"reuse_driver": True,
"keep_drivers_alive": True,
"close_on_crash": True, # When Ready for Production change to True
"headless": True, # When Ready change to True
'output': None, # When Ready for Production Uncomment
}
# max 5 retries add
default_request_options = {
"close_on_crash": True, # When Ready for Production change to True
'output': None, # When Ready for Production Uncomment
"raise_exception":True,
}
================================================
FILE: src/write_output.py
================================================
from botasaurus import bt
from botasaurus.decorators import print_filenames
from .write_output_utils import kebab_case, make_folders
def create_json(path, data):
bt.write_json(data, path, False )
def create_csv(path, data):
bt.write_csv(data, path, False )
def format(query_kebab, entity_type, type):
return f"{query_kebab}.{type}"
def create(data, selected_fields, csv_path, json_path, query_kebab,entity_type):
written = []
fname = json_path + format(query_kebab, entity_type,"json",)
create_json(fname, data)
written.append(fname)
fname = csv_path + format(query_kebab, entity_type,"csv",)
create_csv(fname, data)
written.append(fname)
print_filenames(written)
def write_output(query, data, entity_type,transformer = kebab_case):
query_kebab = transformer(query)
make_folders(query_kebab)
csv_path = f"output/{query_kebab}/csv/"
json_path = f"output/{query_kebab}/json/"
create(data,[], csv_path, json_path, query_kebab,entity_type)
================================================
FILE: src/write_output_utils.py
================================================
from botasaurus.decorators_utils import create_directory_if_not_exists
from casefy import kebabcase
def kebab_case(s):
return kebabcase(s)
def make_folders(query_kebab):
create_directory_if_not_exists(f"output/{query_kebab}/")
create_directory_if_not_exists(f"output/{query_kebab}/json/")
create_directory_if_not_exists(f"output/{query_kebab}/csv/")
def format(query_kebab, type, name):
return f"{name}-of-{query_kebab}.{type}"