config.yaml

# If the scraper does not have a custom setting, use the default
# Any value in the default section can also be set in a specific scraper as well
default:
  dispatch:
    service:
      type: local
    ratelimit:
      type: qps
      value: 1

  downloader:
    save_metadata: true
    save_data:
      service: local
    file_template: "source/{scraper_name}.html"

  extractor:
    save_data:
      service: local
    file_template: "extracted/{scraper_name}.json"

# Custom per scraper
minimal:
  downloader:
    file_template: "source/{scraper_name}/minimal.html"
  extractor:
    file_template: "extracted/{scraper_name}/minimal.json"

multiple_extractors:
  downloader:
    file_template: "source/{scraper_name}/multiple_extractors.html"
  extractor:
    file_template: "extracted/{scraper_name}/multi_test_{extractor_name}.json"

json_source:
  downloader:
    file_template: "source/{scraper_name}/json_source.json"
  extractor:
    file_template: "extracted/{scraper_name}/json_source_data.json"

qa_results:
  downloader:
    file_template: "source/{scraper_name}/qa_source.html"
  extractor:
    file_template: "extracted/{scraper_name}/qa_data.json"

multiple_sources:
  downloader:
    file_template: "source/{scraper_name}/multiple_sources_{source_name}.json"
  extractor:
    file_template: "extracted/{scraper_name}/multiple_sources.json"

page_trigger_download:
  downloader:
    file_template: "source/{scraper_name}/page_{page}.html"
  extractor:
    file_template: "extracted/{scraper_name}/page_{page}.json"

page_dispatch:
  downloader:
    file_template: "source/{scraper_name}/page_{page}.html"
  extractor:
    file_template: "extracted/{scraper_name}/page_{page}.json"

gen_cookie_requests:
  downloader:
    file_template: "source/{scraper_name}/gen_cookie_requests.json"
  extractor:
    file_template: "extracted/{scraper_name}/gen_cookie_requests_data.json"

dispatch_cookie_selenium:
  dispatch:
    ratelimit:
      type: period
      value: 0.017  # ~1 min
  downloader:
    file_template: "source/{scraper_name}/dispatch_cookie_selenium_{ref_id}.json"
  extractor:
    file_template: "extracted/{scraper_name}/dispatch_cookie_selenium_{ref_id}.json"