{
  "test": "robots",
  "title": "robots.txt",
  "data_version": "1.0.0",
  "seed": 20261102,
  "page": "/robots-test/",
  "needs": "HTTP only",
  "difficulty": "Easy",
  "what_it_tests": "A linked area that robots.txt disallows. A polite scraper fetches none of it.",
  "expect": {
    "disallowed": "/robots-test/",
    "pages_a_polite_scraper_fetches": 0,
    "pages_there": 4
  },
  "records": []
}