Powerful Job Monitoring & Scheduling for Web Scraping

## my_scraper.py

# Step 1: Import ScrapeOpsRequests SDK
from scrapeops_python_requests.scrapeops_requests import ScrapeOpsRequests

# Step 2: Initialize SrapeOps Logger
scrapeops_logger =  ScrapeOpsRequests(
  scrapeops_api_key='API_KEY_HERE', 
  spider_name='SPIDER_NAME_HERE',
  job_name='JOB_NAME_HERE',
)

## Step 3: Initialize the ScrapeOps Python Requests Wrapper {#step-3-initialize-the-scrapeops-python-requests-wrapper}
requests = scrapeops_logger.RequestsWrapper() 

urls = ['http://example.com/1/', 'http://example.com/2/']

for url in urls:
    response = requests.get(url)

    ## Step 4: Log Scraped Item
    scrapeops_logger.item_scraped(
        response=response,
        item={'demo': 'hello'}
    )

## settings.py

# Add Your ScrapeOps API key
SCRAPEOPS_API_KEY = 'YOUR_API_KEY'


# Add In The ScrapeOps Extension
EXTENSIONS = {
'scrapeops_scrapy.extension.ScrapeOpsMonitor': 500, 
}


# Update The Download Middlewares
DOWNLOADER_MIDDLEWARES = {
'scrapeops_scrapy.middleware.retry.RetryMiddleware': 550,
'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
}

## DONE! {#done}