Sitemap: https://spectrumnoticias.com/sitemap.xml

User-agent: *

Crawl-delay: 1




# Allowed Paths
Allow: /$ #
Allow: /ny/nyc #
Allow: /ny/nyc/* #
Allow: /tx/texas #
Allow: /tx/texas/* #
Allow: /ca/los-angeles #
Allow: /ca/los-angeles/* #
Allow: /us/noticias #
Allow: /us/noticias/* #
Allow: /fl/florida #
Allow: /fl/florida/* #
Allow: /sitemap.xml #
Allow: /services/* #
Allow: /content/* #
Allow: /etc/* #
Allow: /.well-known/assetlinks.json #
Allow: /local #
Allow: /splash #
Allow: /etc.clientlibs/* #





# Excluded Paths
Disallow: /* #
Disallow: /*/*/partner-content/* #
Disallow: /content/news/stories/* #


# Additional Config
User-Agent: Twitterbot
Disallow: /.well-known/

User-agent: GPTBot 
Disallow: / 

User-agent: anthropic-ai 
Disallow: / 

User-agent: PetalBot 
Disallow: / 

User-agent: Amazonbot 
Disallow: / 

User-agent: ChatGPT 
Disallow: / 

User-agent: Bytespider 
Disallow: / 

User-agent: ClaudeBot 
Disallow: / 

User-agent: PerplexityBot 
Disallow: / 

User-agent: meta-externalagent 
Disallow: / 

User-agent: chatgpt
Disallow: / 

User-agent: CCBot
Disallow: / 

User-agent: Yeti
Disallow: / 

User-agent: Diffbot
Disallow: / 

User-agent: Timpibot
Disallow: / 

User-agent: Ai2Bot
Disallow: / 

User-agent: MistralAI-User
Disallow: / 

User-agent: AnthropicBot
Disallow: / 

User-agent: claude-web
Disallow: / 

User-agent: cohere-ai
Disallow: / 

User-agent: omgili 
Disallow: / 

User-agent: Google-Extended
Disallow: / 

User-agent: Applebot-Extended
Disallow: /