# Allow all search engines to crawl the entire site except for admin or sensitive pages 
User-agent: *
# Disallow common administrative sections and sensitive content
Disallow: /admin/
Disallow: /login/
Disallow: /checkout/
Disallow: /cart/
Disallow: /cgi-bin/
Disallow: /scripts/
Disallow: /temp/
Disallow: /private/
Disallow: /user-data/
Disallow: /order-history/

# Allow search engines to crawl images, videos, assets, and essential content
Allow: /images/
Allow: /videos/
Allow: /assets/
Allow: /uploads/
Allow: /blog/

# Disallow crawling of file formats that don't provide SEO value
Disallow: /*.pdf$
Disallow: /*.doc$
Disallow: /*.xls$
Disallow: /*.ppt$

# Prevent crawling of duplicate content
Disallow: /?*session_id=
Disallow: /?*sort_by=
Disallow: /?*filter=

# Allow search engines to crawl important parts of the site, including pages with structured data
Allow: /products/
Allow: /services/
Allow: /about-us/
Allow: /contact-us/
Allow: /blog/

# Prevent crawling of unimportant or dynamically generated content
Disallow: /search/
Disallow: /tag/
Disallow: /filter/

# Ensure search engines don’t index tracking or sorting parameters
Disallow: /?*filter=
Disallow: /?*sort=
Disallow: /?*page=

# Block specific crawlers (optional) that may be unnecessary or intrusive
User-agent: Baiduspider
Disallow: /

User-agent: Yandex
Disallow: /

# Sitemap Location
Sitemap: https://www.ssnatu.com/sitemap.xml

# Google-specific crawlers and directives
User-agent: Googlebot
Disallow: /checkout/
Disallow: /cart/

# For better indexing of content and understanding of your website's structure
Crawl-delay: 10  # (optional) Avoid overwhelming your server by limiting the crawl rate

# Adding alternate mobile URLs for mobile-first indexing
# Allow Google to discover alternate content types (AMP pages, etc.)
Allow: /amp/