# ============================================================================
# Workzil - robots.txt
# https://www.workzil.com
# Last updated: 2026-05-22
#
# AI Job Hunting Platform - Search, Apply, Rehearse, Track
# ============================================================================
#
# AI / LLM Resources:
#   - LLMs.txt (summary):     https://www.workzil.com/llms.txt
#   - LLMs-full.txt (full):   https://www.workzil.com/llms-full.txt
#   - AI Plugin Manifest:     https://www.workzil.com/.well-known/ai-plugin.json
#   - AI Content API:         https://www.workzil.com/api/ai-content
#   - Discovery Index API:    https://www.workzil.com/api/public/discover.json
#   - Public Jobs Feed:       https://www.workzil.com/api/public/jobs.json
#   - Public Jobs RSS:        https://www.workzil.com/api/public/jobs.rss
#   - Embeddable Jobs Widget: https://www.workzil.com/embed/jobs
#
# Featured Pages:
#   - Homepage:               https://www.workzil.com/
#   - Job Search:             https://www.workzil.com/jobs
#   - AI Interview Trial:     https://www.workzil.com/tools/ai-interview-trial
#   - AI Interview Prep:      https://www.workzil.com/features/ai-interview-prep
#   - Pricing:                https://www.workzil.com/pricing
#   - Blog:                   https://www.workzil.com/blog
#   - For Employers:          https://www.workzil.com/employer/post
#   - Affiliates:             https://www.workzil.com/affiliates-program
#
# Free Tools:
#   - AI Interview Trial:     https://www.workzil.com/tools/ai-interview-trial
#   - Resume Builder:         https://www.workzil.com/resume-builder
#   - Resume Templates:       https://www.workzil.com/templates/resume
#   - Resume Reviewer:        https://www.workzil.com/resume-reviewer
#   - CV Builder:             https://www.workzil.com/cv-builder
#   - CV Reviewer:            https://www.workzil.com/cv-reviewer
#   - Salary Calculator:      https://www.workzil.com/salary-calculator
#   - Salary Comparison:      https://www.workzil.com/salary-comparison
#   - Role Comparison:        https://www.workzil.com/compare-roles
#   - Cover Letter Generator: https://www.workzil.com/tools/cover-letter-generator
#   - Jobs Widget (embed):    https://www.workzil.com/tools/jobs-widget
#
# Company Intelligence + Insights:
#   - Layoff Tracker:         https://www.workzil.com/insights/layoff-tracker
#   - Hot Hiring 200:         https://www.workzil.com/insights/hot-hiring-200
#   - Salary Explorer:        https://www.workzil.com/insights/salary-explorer
#   - Visa Sponsorship:       https://www.workzil.com/insights/visa-sponsorship
#   - Hiring Trends:          https://www.workzil.com/hiring-trends
#   - Funding Tracker:        https://www.workzil.com/funding
#   - Expansion Tracker:      https://www.workzil.com/expansion
#   - Layoff News:            https://www.workzil.com/layoff-news
#   - Best Companies:         https://www.workzil.com/best-companies-to-work-for
#
# Listicle Pillars:
#   - Best AI Job Search Tools 2026:
#     https://www.workzil.com/insights/best-ai-job-search-tools-2026
#   - Best Auto-Apply Tools 2026:
#     https://www.workzil.com/insights/best-auto-apply-tools-2026
#   - Best Job Tracker Tools 2026:
#     https://www.workzil.com/insights/best-job-tracker-tools-2026
#   - Best AI Cover Letter Tools 2026:
#     https://www.workzil.com/insights/best-ai-cover-letter-tools-2026
#   - Best Chrome Extensions 2026:
#     https://www.workzil.com/insights/best-job-search-chrome-extensions-2026
#
# Programmatic SEO families (~4,000 pages):
#   - /companies/<slug>                          175 layoff-relevant seed
#   - /companies/<slug>/layoffs                  Per-company layoff history
#   - /visa-sponsorship/h1b/<company>            60 top H-1B sponsors
#   - /alternatives/<competitor>                 20 competitor alternatives
#   - /salary/<role>                             148 roles
#   - /salary/<role>/<country>                   ~2,500 role x country combos
#   - /remote-jobs/<role>                        148 roles
#   - /jobs/<role>                               148 roles
#   - /jobs/<role>/in/<city>                     300+ role x city combos
#   - /interview-questions/<role>                148 roles + AI trial widget
#   - /templates/resume/<template>               16 resume templates
#
# Career Research:
#   - Industry Guides:        https://www.workzil.com/industry/technology
#   - Market Intelligence:    https://www.workzil.com/market-intelligence
#   - Resources:              https://www.workzil.com/resources
# ============================================================================


# ============================================================================
# DEFAULT RULES - All well-behaved crawlers
# ============================================================================
User-agent: *
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/
Allow: /api/ai-content

# Public Career Pages (Ashby alternative — Google for Jobs ingests these)
Allow: /c/

# Public discovery + data surfaces — CORS-open, no auth required.
# Designed for AI agents, aggregators, and embeds.
Allow: /api/public/discover.json
Allow: /api/public/jobs.json
Allow: /api/public/jobs.rss
Allow: /embed/jobs

# Application routes - private
Disallow: /api/
Disallow: /dashboard/
Disallow: /employer/
Allow: /employer/post
Disallow: /onboarding/
Disallow: /admin/
Disallow: /_next/
Disallow: /static/

# Auth routes
Disallow: /sign-in
Disallow: /sign-in/
Disallow: /sign-up
Disallow: /sign-up/
Disallow: /verify-email
Disallow: /verify-email/
Disallow: /sso-callback/

# Dynamic route patterns
Disallow: /*[slug]*
Disallow: /*[category]*
Disallow: /*[id]*
Disallow: /*[...]*

# Query string spam prevention
Disallow: /*?ref=
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?page=0
Disallow: /*?preview=

# Bulk-harvest guards — discourage deep paginated/parametrised dataset crawling.
# Canonical no-param pages stay fully allowed (these only block ?limit= / ?offset=
# enumeration). The feeds themselves hard-cap limit server-side regardless.
Disallow: /*?limit=
Disallow: /*?offset=


# ============================================================================
# HONEYPOT TRAP DIRECTORIES
# These paths do not exist. Any bot crawling them is likely malicious.
# Used for bot detection and security monitoring.
# ============================================================================
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /wp-config.php
Disallow: /administrator/
Disallow: /admin.php
Disallow: /phpmyadmin/
Disallow: /pma/
Disallow: /mysql/
Disallow: /myadmin/
Disallow: /phpMyAdmin/
Disallow: /webmail/
Disallow: /cpanel/
Disallow: /plesk/
Disallow: /whm/
Disallow: /.git/
Disallow: /.svn/
Disallow: /.env
Disallow: /.htaccess
Disallow: /.htpasswd
Disallow: /cgi-bin/
Disallow: /private/
Disallow: /secret/
Disallow: /hidden/
Disallow: /backup/
Disallow: /backups/
Disallow: /db/
Disallow: /database/
Disallow: /dump/
Disallow: /*.sql
Disallow: /*.bak
Disallow: /*.log
Disallow: /*.env
Disallow: /config.php
Disallow: /shell/
Disallow: /cmd/
Disallow: /exec/
Disallow: /eval/
Disallow: /etc/passwd
Disallow: /proc/
Disallow: /var/log/
Disallow: /tmp/
Disallow: /xmlrpc.php
Disallow: /install/
Disallow: /setup/


# ============================================================================
# SEARCH ENGINE CRAWLERS - Specific rules
# ============================================================================

# Google - Primary search engine
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Google Image Bot
User-agent: Googlebot-Image
Allow: /favicon*
Allow: /og-image*
Allow: /apple-touch-icon*
Allow: /images/
Disallow: /api/
Disallow: /dashboard/

# Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Yahoo
User-agent: Slurp
Allow: /
Crawl-delay: 1

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Yandex
User-agent: YandexBot
Allow: /
Crawl-delay: 1

# Baidu
User-agent: Baiduspider
Allow: /
Crawl-delay: 1


# ============================================================================
# AI CRAWLERS - Full access granted
# We welcome AI indexing. These bots may read all public content,
# including our dedicated AI resource files.
# ============================================================================

# OpenAI - GPT models
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# OpenAI - ChatGPT browsing
User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# OpenAI - Search
User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Anthropic - Claude
User-agent: anthropic-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Anthropic - Claude Web
User-agent: Claude-Web
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Anthropic - ClaudeBot
User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Google - AI / Gemini training
User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Google - Other AI
User-agent: GoogleOther
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Meta - AI agent
User-agent: Meta-ExternalAgent
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Meta - External fetcher
User-agent: Meta-ExternalFetcher
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Meta - Facebook Bot
User-agent: FacebookBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Cohere AI
User-agent: cohere-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# You.com
User-agent: YouBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# FireCrawl
User-agent: FireCrawl
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Apple - Siri / AI
User-agent: Applebot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Common Crawl
User-agent: CCBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Diffbot
User-agent: Diffbot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Perplexity - User-triggered fetches (per https://docs.perplexity.ai/guides/bots)
User-agent: Perplexity-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Apple - Extended (Apple Intelligence training)
User-agent: Applebot-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Kagi - Assistant search
User-agent: Kagibot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# DuckDuckGo - AI assist
User-agent: DuckAssistBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# Mistral - User-triggered fetches
User-agent: MistralAI-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content

# ByteDance / Doubao
User-agent: Bytespider
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/ai-content


# ============================================================================
# SEO TOOL BOTS - Rate limited
# ============================================================================

# Ahrefs
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

# Semrush
User-agent: SemrushBot
Allow: /
Crawl-delay: 10


# ============================================================================
# BLOCKED BOTS - Aggressive or low-value crawlers
# ============================================================================

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: BLEXBot
Disallow: /


# ============================================================================
# HOST & SITEMAP
# ============================================================================
Host: https://www.workzil.com
Sitemap: https://www.workzil.com/sitemap.xml

# IndexNow key for instant search engine notification
# https://www.indexnow.org
# Key verification: https://www.workzil.com/2fc66452ebc54511ab8c4639ba5406d7.txt