# Allow all crawlers by default User-agent: * Allow: / # Block admin and authentication areas Disallow: /admin/ Disallow: /admins/ Disallow: /hr/admins/ Disallow: /en/admins/ # Block internal API endpoints Disallow: /tinymce_assets # Block attachment file management Disallow: /attachment_files # Block form validation endpoints Disallow: */validate # Block search results (allow search page itself) Disallow: /search? Disallow: /hr/search? Disallow: /en/search? # Allow everything else Allow: /