Sitemap: https://www.actowizmetrics.com/sitemap.xml
Sitemap: https://www.actowizmetrics.com/urllist.txt
Sitemap: https://www.actowizmetrics.com/ror.xml

# =========================
# DEFAULT RULE FOR ALL CRAWLERS
# =========================
User-agent: *

# =========================
# ALLOW IMPORTANT RESOURCES
# =========================
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /assets/
Allow: /uploads/

# =========================
# BLOCK PRIVATE / SYSTEM AREAS
# =========================
Disallow: /thank-you.php
Disallow: /userdata.php
Disallow: /dsadmin/
Disallow: /iFrame-Dashboard.html
Disallow: /tab-common.php
Disallow: /analytics/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /cache/
Disallow: /includes/
Disallow: /vendor/
Disallow: /logs/
Disallow: /backup/
Disallow: /config/
Disallow: /cron/
Disallow: /test/
Disallow: /dev/

# =========================
# PREVENT URL PARAMETERS INDEXING
# =========================
Disallow: /*?*
Allow: /*?page=
Allow: /*?search=

# =========================
# SEARCH ENGINES
# =========================

User-agent: Googlebot
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

User-agent: AdsBot-Google
Allow: /

User-agent: Bingbot
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

# =========================
# AI / LLM / SEARCH CRAWLERS
# =========================

# OpenAI
User-agent: GPTBot
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

User-agent: OAI-SearchBot
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

User-agent: ChatGPT-User
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

# Anthropic
User-agent: ClaudeBot
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

User-agent: anthropic-ai
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

# Google AI
User-agent: Google-Extended
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

# Common Crawl
User-agent: CCBot
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

# Amazon AI
User-agent: Amazonbot
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

# Bytespider (TikTok AI/Search)
User-agent: Bytespider
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

# You.com AI
User-agent: YouBot
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

# Apple AI
User-agent: Applebot
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

User-agent: Applebot-Extended
Allow: /
Disallow: /thank-you.php
Disallow: /userdata.php

# =========================
# CRAWL SETTINGS
# =========================
Crawl-delay: 2