User-agent: * # Only allow these 2 pages for public crawling Allow: /$ Allow: /login # Allow js and css so bot can render pages client side Allow: /*.js Allow: /*.css # Disallow everything else Disallow: /