# Karostartup — robots.txt
# https://www.robotstxt.org/

User-agent: *
Allow: /

# Auth flows and the admin CMS aren't useful in search results
Disallow: /admin/
Disallow: /auth/
Disallow: /profile.html

# The search page is served with <meta name="robots" content="noindex,follow">.
# It must stay CRAWLABLE — a Disallow here would stop crawlers ever reading that
# noindex, which is what actually keeps search-results pages out of the index.

# Don't crawl raw query-string variants of the contact form
Disallow: /contact.html?

# No Crawl-delay: this is a news site and throttling Bingbot/Yandex only slows
# down discovery of freshly published articles.

Sitemap: https://www.karostartup.com/sitemap.xml
Sitemap: https://www.karostartup.com/news-sitemap.xml