# robots.txt for AllOneTools.com
# Version 1.0 - Last updated: 2025-04-18

User-agent: *
Allow: /

# Allow crawling of all tool pages
Allow: /dev-tools/
Allow: /creative-tools/
Allow: /tools/

# Sitemap location
Sitemap: https://allonetools.com/sitemap.xml

# Block common bots that might overload the server
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

# Allow Googlebot full access
User-agent: Googlebot
Allow: /

# Allow Bingbot full access
User-agent: Bingbot
Allow: /

# Allow structured data testing tools
User-agent: Twitterbot
Allow: /
User-agent: facebookexternalhit
Allow: /

# Disallow admin and development resources if they exist
Disallow: /admin/
Disallow: /dev/
Disallow: /api/
Disallow: /internal/
Disallow: /cdn-cgi/

# Allow specific file types
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.gif$
Allow: /*.svg$

# Allow specific paths
Allow: /dev-tools/
Allow: /creative-tools/
Allow: /about
Allow: /contact
Allow: /privacy
Allow: /terms
Allow: /cookies

# Block certain parameters
Disallow: /*?*
Disallow: /*.json$
Disallow: /*.xml$
Disallow: /*.log$

# Block specific paths
Disallow: /search
Disallow: /login
Disallow: /register
Disallow: /forgot-password

# Host directive
Host: allonetools.com

# Clean parameters
Clean-param: ref /search/
Clean-param: utm_source&utm_medium&utm_campaign&utm_term&utm_content /

# Allow main pages
Allow: /
Allow: /about
Allow: /contact
Allow: /privacy
Allow: /terms
Allow: /cookies
Allow: /sitemap

# Allow all tool pages
# JSON Tools
Allow: /dev-tools/json-formatter
Allow: /dev-tools/json-validator
Allow: /dev-tools/json-minifier
Allow: /dev-tools/json-converter

# HTML Tools
Allow: /dev-tools/html-formatter
Allow: /dev-tools/html-validator
Allow: /dev-tools/html-minifier
Allow: /dev-tools/html-converter

# XML Tools
Allow: /dev-tools/xml-formatter
Allow: /dev-tools/xml-minifier
Allow: /dev-tools/xml-validator
Allow: /dev-tools/xml-converter

# Image Tools
Allow: /creative-tools/image-compressor
Allow: /creative-tools/image-upscaler
Allow: /creative-tools/background-remover

# SEO Tools
Allow: /creative-tools/meta-tag-generator