# robots.txt — navlyt.com
#
# One group for every crawler. The previous file gave each AI crawler its own
# "Allow: /" group, and under RFC 9309 a crawler obeys only the most specific
# group naming it and ignores "User-agent: *" — so GPTBot, OAI-SearchBot,
# ClaudeBot, PerplexityBot and the rest were never told about the app routes
# below. Listing them here keeps them explicitly welcome on public content and
# gives them the same private-route rules as everyone else.
#
# Search vs training are separate controls and are kept separate:
#   OAI-SearchBot    — ChatGPT search results (discovery)
#   ChatGPT-User     — fetches a page a ChatGPT user asked about
#   GPTBot           — OpenAI model training
#   Claude-SearchBot — Claude search results; Claude-User — user-requested fetch
#   ClaudeBot        — Anthropic model training
#   PerplexityBot    — Perplexity search index; Perplexity-User — user fetch
#   Google-Extended  — Gemini training/grounding; does not affect Google Search
# Training crawlers (GPTBot, ClaudeBot, Google-Extended) are allowed on public
# pages because that was this site's existing policy. Changing it is an owner
# decision; scripts/check-robots.mjs pins it so it cannot change by accident.

User-agent: *
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
Allow: /

# Authenticated application routes: no public content, and nothing a crawler
# should request. The data itself is protected by auth and row-level security;
# this keeps crawlers from spending requests on login walls.
Disallow: /dashboard
Disallow: /documents
# Only the app sub-routes; the marketing /compliance page stays crawlable.
Disallow: /compliance/tasks
Disallow: /compliance/calendar
Disallow: /crew
Disallow: /aircraft
Disallow: /tripdesk
Disallow: /reports
Disallow: /settings
Disallow: /requirements
Disallow: /corrective-actions
Disallow: /regulatory-changes
Disallow: /reset-password
# Auditor share links carry an access token in the path. Trailing slash on
# purpose: /aviation-audit-software is a public page and must not match.
Disallow: /audit/

Sitemap: https://www.navlyt.com/sitemap.xml