# robots.txt for ArgoRobots
# Last updated: 2026-05-31

# Allow all search engines to crawl the site
User-agent: *
Allow: /

# Disallow admin areas
Disallow: /admin/

# Disallow API endpoints (return JSON, not useful for indexing)
Disallow: /api/

# Disallow user profile pages in community
Disallow: /community/users/

# Disallow database directory
Disallow: /database/

# Disallow resources directory (backend/data files), but allow the
# render-critical assets. Google needs CSS, JS, and images to render pages
# correctly when evaluating layout and mobile-friendliness.
Disallow: /resources/
Allow: /resources/styles/
Allow: /resources/scripts/
Allow: /resources/images/

# Disallow error pages
Disallow: /error-pages/

# Disallow specific file types
Disallow: /*.sqlite$
Disallow: /*.sql$
Disallow: /composer.json
Disallow: /composer.lock

# Crawl delay (optional - can be adjusted or removed)
# Crawl-delay: 1

# Sitemap location
Sitemap: https://argorobots.com/sitemap.xml