Web scraping indeed.com

import json
from parsel import Selector

# install using `pip install scraperapi`
from scraper_api import ScraperAPIClient

# create an API client instance
client = ScraperAPIClient(api_key="YOUR API KEY")

# create scrape function that returns HTML parser for a given URL
def scrape(url: str, country: str="", render_js=False, headers: dict=None) -> Selector:
    api_result = client.get(
        url=url,
        headers=headers or {},
        render=False,
        url=https://www.indeed.com/viewjob?jk=89a7a5713f2d78c0,
        method=GET,
    )
    assert api_result.ok, api_result.text
    return Selector(api_result.text)

# example search page url:
url = "https://www.indeed.com/jobs?q=python&l=Seattle%2C%20WA"
selector = scrape(url, country="US")

# Indeed jobs can be found in Javascript variable as an array of job objects:
data = selector.re(r'window.mosaic.providerData\["mosaic-provider-jobcards"\]=(\{.+?\});')
data = json.loads(data[0])
jobs = data["metaData"]["mosaicProviderJobCardsModel"]["results"]
print(len(jobs))
15
from pprint import pprint
pprint(jobs[0])

import json
from parsel import Selector

# install using `pip install zenrows`
from zenrows import ZenRowsClient

# create an API client instance
client = ZenRowsClient(apikey="YOUR API KEY")

# create scrape function that returns HTML parser for a given URL
def scrape(url: str, country: str="", render_js=False, headers: dict=None) -> Selector:
    api_result = client.get(
        url,
        headers=headers,
        params={
            "json_response": "True",
            "js_render": "True",
            "premium_proxy": "True",
            "url": "https://www.indeed.com/viewjob?jk=89a7a5713f2d78c0",
            "method": "GET",
        }
    )
    assert api_result.ok, api_result.text
    data = api_result.json()
    return Selector(data['html'])

# example search page url:
url = "https://www.indeed.com/jobs?q=python&l=Seattle%2C%20WA"
selector = scrape(url, country="US")

# Indeed jobs can be found in Javascript variable as an array of job objects:
data = selector.re(r'window.mosaic.providerData\["mosaic-provider-jobcards"\]=(\{.+?\});')
data = json.loads(data[0])
jobs = data["metaData"]["mosaicProviderJobCardsModel"]["results"]
print(len(jobs))
15
from pprint import pprint
pprint(jobs[0])

import json
from parsel import Selector

# webscrapingapi has a Python SDK but it's not great, use httpx instead:
# `pip install httpx`
import httpx

# create an API client instance
client = httpx.Client(timeout=180)

# create scrape function that returns HTML parser for a given URL
def scrape(url: str, country: str="", render_js=False, headers: dict=None) -> Selector:
    api_result = client.get(
        url,
        headers=headers,
        params={
            "url": url,
            "api_key": "YOUR API KEY",  # NOTE: add your API KEY here!
            "timeout": 60_000,
            "render_js": "False",
            "url": "https://www.indeed.com/viewjob?jk=89a7a5713f2d78c0",
            "method": "GET",
        },
    )
    assert api_result.status_code == 200, api_result.reason_phrase
    return Selector(api_result.text)

# example search page url:
url = "https://www.indeed.com/jobs?q=python&l=Seattle%2C%20WA"
selector = scrape(url, country="US")

# Indeed jobs can be found in Javascript variable as an array of job objects:
data = selector.re(r'window.mosaic.providerData\["mosaic-provider-jobcards"\]=(\{.+?\});')
data = json.loads(data[0])
jobs = data["metaData"]["mosaicProviderJobCardsModel"]["results"]
print(len(jobs))
15
from pprint import pprint
pprint(jobs[0])

import json
from parsel import Selector

# install using `pip install scrapingbee`
from scrapingbee import ScrapingBeeClient

# create an API client instance
client = ScrapingBeeClient(api_key="YOUR API KEY")

# create scrape function that returns HTML parser for a given URL
def scrape(url: str, country: str="", render_js=False, headers: dict=None) -> Selector:
    api_result = client.get(
        url,
        headers=headers,
        params={
            "json_response": True,
            "transparent_status_code": True,
        }
    )
    assert api_result.ok, api_result.text
    data = api_result.json()
    return Selector(data['body'])

# example search page url:
url = "https://www.indeed.com/jobs?q=python&l=Seattle%2C%20WA"
selector = scrape(url, country="US")

# Indeed jobs can be found in Javascript variable as an array of job objects:
data = selector.re(r'window.mosaic.providerData\["mosaic-provider-jobcards"\]=(\{.+?\});')
data = json.loads(data[0])
jobs = data["metaData"]["mosaicProviderJobCardsModel"]["results"]
print(len(jobs))
15
from pprint import pprint
pprint(jobs[0])

import json
from parsel import Selector

# install using `pip install scrapingant-client`
from scrapingant_client import ScrapingAntClient

# create an API client instance
client = ScrapingAntClient(token="YOUR API KEY")

# create scrape function that returns HTML parser for a given URL
def scrape(url: str, country: str="", render_js=False, headers: dict=None) -> Selector:
    api_result = client.general_request(
        url,
        json=True,
        proxy_type='residential',
        url='https://www.indeed.com/viewjob?jk=89a7a5713f2d78c0',
        method='GET',
    )
    assert api_result.ok, api_result.text
    return Selector(api_result.text)

# example search page url:
url = "https://www.indeed.com/jobs?q=python&l=Seattle%2C%20WA"
selector = scrape(url, country="US")

# Indeed jobs can be found in Javascript variable as an array of job objects:
data = selector.re(r'window.mosaic.providerData\["mosaic-provider-jobcards"\]=(\{.+?\});')
data = json.loads(data[0])
jobs = data["metaData"]["mosaicProviderJobCardsModel"]["results"]
print(len(jobs))
15
from pprint import pprint
pprint(jobs[0])

import json
from parsel import Selector

# scrapingdog has no integration but we can use httpx
# install using `pip install httpx`
import httpx

# create an API client instance
client = httpx.Client(timeout=180)

# create scrape function that returns HTML parser for a given URL
def scrape(url: str, country: str="", render_js=False, headers: dict=None) -> Selector:
    payload = {
        "api_key": "YOUR API KEY",
        "url": url,

    }
    api_result = client.post(
        "https://api.scrapingdog.com/scrape",
        json=payload,
    )
    data = api_result.json()
    assert data['success'], f"scrape failed: {data['message']}"
    return Selector(data['html'])

# example search page url:
url = "https://www.indeed.com/jobs?q=python&l=Seattle%2C%20WA"
selector = scrape(url, country="US")

# Indeed jobs can be found in Javascript variable as an array of job objects:
data = selector.re(r'window.mosaic.providerData\["mosaic-provider-jobcards"\]=(\{.+?\});')
data = json.loads(data[0])
jobs = data["metaData"]["mosaicProviderJobCardsModel"]["results"]
print(len(jobs))
15
from pprint import pprint
pprint(jobs[0])

import json
from parsel import Selector

# create an API client instance
# create scrape function that returns HTML parser for a given URL
def scrape(url: str, country: str="", render_js=False, headers: dict=None) -> Selector:
    # example search page url:
    url = "https://www.indeed.com/jobs?q=python&l=Seattle%2C%20WA"
    selector = scrape(url, country="US")
    # Indeed jobs can be found in Javascript variable as an array of job objects:
    data = selector.re(r'window.mosaic.providerData\["mosaic-provider-jobcards"\]=(\{.+?\});')
    data = json.loads(data[0])
    jobs = data["metaData"]["mosaicProviderJobCardsModel"]["results"]
    print(len(jobs))
    15
    from pprint import pprint
    pprint(jobs[0])