# Allow all robots to access the site
User-agent: *
Allow: /

# Disallow indexing of specific folders
Disallow: /admin/
Disallow: /cgi-bin/
Disallow: /temp/
Disallow: /private/
Disallow: /scripts/
Disallow: /login/
Disallow: /register/
Disallow: /reset-password/
Disallow: /cart/
Disallow: /checkout/
Disallow: /search/

# Block specific file types
Disallow: /*.php$
Disallow: /*.cgi$
Disallow: /*.pl$
Disallow: /*.py$

# Allow search engines to index these specific pages
Allow: /$
Allow: /about-us/
Allow: /services/
Allow: /contact-us/
Allow: /blog/
Allow: /marine-digital-marketing/
Allow: /portfolio/
Allow: /online-reputation-management/
Allow: /our-work/
Allow: /search-engine-optimization/
Allow: /branding/
Allow: /e-commerce-design-development/
Allow: /real-estate-digital-marketing/
Allow: /digital-marketing-services/
Allow: /mobile-app-design-development/
Allow: /web-design-development/
Allow: /privacy-policy-2/
Allow: /web-mobile-app/
Allow: /outdoor-advertising/
Allow: /content-marketing/
Allow: /social-media-marketing/
Allow: /email-marketing-services/
Allow: /terms-conditions/


# Specify the location of the sitemap
Sitemap: https://www.webhooters.com/sitemap.xml
Sitemap: https://www.webhooters.com/page-sitemap.xml