User-agent: * Allow: /* #allows to crawl all levels of url Sitemap: https://www.knowledgehut.com/sitemap.xml Allow: /events/* Allow: /blog/* #allows to crawl blog Allow: /*.xml Allow: /tutorials/* Allow: /practice_tests/*/Questions/*= Disallow: /auth/* #Restrcicts crawling Auth urls Disallow: /*? #Restrcicts crawling of URL containg "?" Disallow: /*= #Restrcicts crawling of URL containg "=" Disallow: /*_ #Restrcicts crawling of URL containg "_" Disallow: /*payment #Restrcicts crawling of URL containg payment term Disallow: /*.php Disallow: /*.index Disallow: /home Disallow: /blog/*? #blocks crawling of url containg "?" Disallow: /wp-blog* Disallow: /kh-prepo/* #Restrcicts crawling Kh-Prepo Urls Disallow: /submit-review/* #Restrcicts crawling Submit Review URLs Disallow: /pay/* #Restrcicts crawling pay URLs Disallow: /lp/ Disallow: /drip/ Disallow: /blog/[category]/[slug] Disallow: /blog/category/[slug] Disallow: /blog/author/[slug] Disallow: /blog/subscribe/[...index] Disallow: /blog/tag/[slug] Disallow: /blog/type/[slug] Disallow: /online Disallow: /online/*