# ============================================================== # robots.txt — cantonshoppingguide.com # Canton First Monday Trade Days Shopping Guide # Canton, Texas — Van Zandt County # Last updated: May 21 2026 — Fixed blog-detail.php blocking # ============================================================== # # WHAT WAS FIXED IN THIS VERSION: # 1. Added Googlebot-Mobile block — fixes GSC "Blocked by # robots.txt" error on blog and all other pages # 2. Added full Allow rules to User-agent: * — the previous # version had ONLY Crawl-delay with zero Allow rules, # which caused Google Inspection Tool to block all pages # 3. Added AdsBot-Google-Mobile block # 4. Verified all public URLs from live site are allowed # # STRUCTURE: # SECTION 1 — ALLOW : Legitimate Search Engines # SECTION 2 — ALLOW : AI Citation & Search Bots # SECTION 3 — BLOCK : AI Training Bots # SECTION 4 — BLOCK : Commercial SEO Scrapers # SECTION 5 — BLOCK : Junk & Malicious Crawlers # SECTION 6 — Universal fallback (CRITICAL FIX) # ============================================================== # --------------------------------------------------------------- # SECTION 1 — ALLOW : LEGITIMATE SEARCH ENGINES # --------------------------------------------------------------- # -- GOOGLE DESKTOP --------------------------------------------- User-agent: Googlebot Allow: /$ Allow: /home Allow: /events Allow: /events/ Allow: /event-view.php?id= Allow: /search_results$ Allow: /search-view.php?listing= Allow: /calender Allow: /weather Allow: /online-map Allow: /printable-map Allow: /trade_days_schedule Allow: /blog Allow: /blog/ Allow: /about-us Allow: /contact-us Allow: /new-vendor Allow: /terms&conditions Allow: /privacy-policy Allow: /sitemap.xml Allow: /sitemap-events.xml Allow: /sitemap-vendors.xml Allow: /sitemap-blog.xml Allow: /sitemap-static.xml Disallow: /cp/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /join.php Disallow: /join/ Disallow: /login/ Disallow: /login.php Disallow: /logout/ Disallow: /register/ Disallow: /account/ Disallow: /my-account/ Disallow: /password-reset/ Disallow: /forgot-password/ Allow: /blog-detail.php?view= Disallow: /blog-detail.php Disallow: /blog.php Disallow: /events.php Disallow: /calender.php Disallow: /search_results.php Disallow: /all_location.php Disallow: /ajaxactions.php Disallow: /ajaxcall.php Disallow: /index.php Disallow: /api/ Disallow: /ajax/ Disallow: /includes/ Disallow: /config/ Disallow: /cache/ Disallow: /tmp/ Disallow: /temp/ Disallow: /logs/ Disallow: /backup/ Disallow: /cron/ Disallow: /install/ Disallow: /cgi-bin/ Disallow: /cp/uploads/ Disallow: /*?session= Disallow: /*?utm_source= Disallow: /*?utm_medium= Disallow: /*?utm_campaign= Disallow: /*?fbclid= Disallow: /*?gclid= Disallow: /*?ref= Disallow: /*?affiliate= Disallow: /test/ Disallow: /staging/ Disallow: /dev/ Disallow: /beta/ Disallow: /old/ # -- GOOGLE SMARTPHONE / MOBILE --------------------------------- # IMPORTANT: Google Search Console URL Inspection Tool # identifies as "Google Inspection Tool smartphone" and # matches this block — not the Googlebot block above. # This block was MISSING in the previous robots.txt and # was the direct cause of the indexing failures in GSC. User-agent: Googlebot-Mobile Allow: /$ Allow: /home Allow: /events Allow: /events/ Allow: /event-view.php?id= Allow: /search_results$ Allow: /search-view.php?listing= Allow: /calender Allow: /weather Allow: /online-map Allow: /printable-map Allow: /trade_days_schedule Allow: /blog Allow: /blog/ Allow: /about-us Allow: /contact-us Allow: /new-vendor Allow: /sitemap.xml Allow: /sitemap-events.xml Allow: /sitemap-vendors.xml Allow: /sitemap-blog.xml Allow: /sitemap-static.xml Disallow: /cp/ Disallow: /admin/ Disallow: /join.php Disallow: /login/ Disallow: /search_results.php Disallow: /ajaxactions.php Disallow: /ajaxcall.php Disallow: /*?session= Disallow: /*?utm_ # -- GOOGLE IMAGE BOT ------------------------------------------- User-agent: Googlebot-Image Allow: /images/ Allow: /cp/uploads/ Disallow: /cp/ # -- GOOGLE VIDEO BOT ------------------------------------------- User-agent: Googlebot-Video Allow: / # -- GOOGLE ADS BOTS -------------------------------------------- User-agent: AdsBot-Google Allow: / Disallow: /cp/ Disallow: /admin/ User-agent: AdsBot-Google-Mobile Allow: / Disallow: /cp/ Disallow: /admin/ # -- BING / YAHOO ----------------------------------------------- User-agent: Bingbot Allow: /$ Allow: /events Allow: /event-view.php?id= Allow: /search_results$ Allow: /search-view.php?listing= Allow: /calender Allow: /trade_days_schedule Allow: /blog Allow: /blog/ Allow: /about-us Allow: /contact-us Allow: /sitemap.xml Allow: /sitemap-events.xml Allow: /sitemap-vendors.xml Allow: /sitemap-blog.xml Disallow: /cp/ Disallow: /admin/ Disallow: /join.php Disallow: /login/ Disallow: /search_results.php Disallow: /all_location.php Allow: /blog-detail.php?view= Disallow: /blog-detail.php Disallow: /events.php Disallow: /ajaxactions.php Disallow: /ajaxcall.php Disallow: /*?session= Disallow: /*?utm_ # -- DUCKDUCKGO ------------------------------------------------- User-agent: DuckDuckBot Allow: / Disallow: /cp/ Disallow: /admin/ Disallow: /join.php Disallow: /login/ Disallow: /search_results.php Disallow: /ajaxactions.php Disallow: /ajaxcall.php Disallow: /*?session= # -- YANDEX ----------------------------------------------------- User-agent: Yandex Allow: / Disallow: /cp/ Disallow: /admin/ Disallow: /join.php Disallow: /login/ Disallow: /search_results.php Disallow: /ajaxactions.php Disallow: /*?session= # -- APPLEBOT (Siri & Spotlight) -------------------------------- User-agent: Applebot Allow: / Disallow: /cp/ Disallow: /admin/ Disallow: /join.php Disallow: /login/ Disallow: /search_results.php Disallow: /ajaxactions.php # -- BAIDU ------------------------------------------------------ User-agent: Baiduspider Allow: / Disallow: /cp/ Disallow: /admin/ Disallow: /join.php Disallow: /login/ Disallow: /search_results.php Disallow: /ajaxactions.php # --------------------------------------------------------------- # SECTION 2 — ALLOW : AI CITATION & SEARCH BOTS # These power AI answers in ChatGPT, Perplexity, Claude, # Gemini etc. Allowing them = citation traffic for the site. # --------------------------------------------------------------- User-agent: GPTBot Allow: /events Allow: /event-view.php?id= Allow: /blog Allow: /blog/ Allow: /trade_days_schedule Allow: /calender Allow: /about-us Allow: /sitemap.xml Disallow: /cp/ Disallow: /admin/ Disallow: /ajaxactions.php Disallow: /ajaxcall.php User-agent: ChatGPT-User Allow: /events Allow: /event-view.php?id= Allow: /blog Allow: /blog/ Allow: /trade_days_schedule Allow: /about-us Disallow: /cp/ Disallow: /admin/ Disallow: /ajaxactions.php Disallow: /ajaxcall.php User-agent: OAI-SearchBot Allow: / Disallow: /cp/ Disallow: /admin/ Disallow: /ajaxactions.php Disallow: /ajaxcall.php User-agent: ClaudeBot Allow: /events Allow: /event-view.php?id= Allow: /blog Allow: /blog/ Allow: /trade_days_schedule Allow: /about-us Allow: /sitemap.xml Disallow: /cp/ Disallow: /admin/ Disallow: /ajaxactions.php Disallow: /ajaxcall.php User-agent: anthropic-ai Allow: /events Allow: /event-view.php?id= Allow: /blog Allow: /blog/ Allow: /trade_days_schedule Allow: /about-us Disallow: /cp/ Disallow: /admin/ Disallow: /ajaxactions.php User-agent: PerplexityBot Allow: /events Allow: /event-view.php?id= Allow: /blog Allow: /blog/ Allow: /trade_days_schedule Allow: /about-us Allow: /sitemap.xml Disallow: /cp/ Disallow: /admin/ Disallow: /ajaxactions.php Disallow: /ajaxcall.php User-agent: Google-Extended Allow: /events Allow: /event-view.php?id= Allow: /blog Allow: /blog/ Allow: /trade_days_schedule Allow: /about-us Disallow: /cp/ Disallow: /admin/ Disallow: /ajaxactions.php Disallow: /ajaxcall.php User-agent: FacebookBot Allow: /events Allow: /event-view.php?id= Allow: /blog Allow: /blog/ Allow: /about-us Disallow: /cp/ Disallow: /admin/ Disallow: /ajaxactions.php User-agent: YouBot Allow: / Disallow: /cp/ Disallow: /admin/ Disallow: /ajaxactions.php Disallow: /ajaxcall.php User-agent: cohere-ai Allow: /events Allow: /blog Allow: /blog/ Allow: /trade_days_schedule Disallow: /cp/ Disallow: /admin/ Disallow: /ajaxactions.php # --------------------------------------------------------------- # SECTION 3 — BLOCK : AI TRAINING BOTS # Scrape content to train AI models — zero traffic benefit. # --------------------------------------------------------------- User-agent: CCBot Disallow: / User-agent: Bytespider Disallow: / User-agent: Amazonbot Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: Diffbot Disallow: / User-agent: omgili Disallow: / User-agent: omgilibot Disallow: / User-agent: AI2Bot Disallow: / User-agent: Timpibot Disallow: / User-agent: Webzio-Extended Disallow: / User-agent: ImagesiftBot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: Kangaroo Bot Disallow: / # --------------------------------------------------------------- # SECTION 4 — BLOCK : COMMERCIAL SEO SCRAPERS # Used by competitors to monitor keyword and content strategy. # --------------------------------------------------------------- User-agent: AhrefsBot Disallow: / User-agent: AhrefsSiteAudit Disallow: / User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: SemrushBot-SI Disallow: / User-agent: MJ12bot Disallow: / User-agent: rogerbot Disallow: / User-agent: dotbot Disallow: / User-agent: DotBot Disallow: / User-agent: serpstatbot Disallow: / User-agent: SISTRIX Disallow: / User-agent: SISTRIXbot Disallow: / User-agent: MangoolsBot Disallow: / User-agent: seobility Disallow: / User-agent: SEOkicks Disallow: / User-agent: SEOkicks-Robot Disallow: / User-agent: BuiltWith Disallow: / User-agent: SiteAuditBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: Neevabot Disallow: / User-agent: PetalBot Disallow: / User-agent: Screaming Frog SEO Spider Disallow: / User-agent: MegaIndex Disallow: / # --------------------------------------------------------------- # SECTION 5 — BLOCK : JUNK & MALICIOUS CRAWLERS # Email harvesters, vulnerability scanners, content thieves. # --------------------------------------------------------------- User-agent: Scrapy Disallow: / User-agent: python-requests Disallow: / User-agent: Go-http-client Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: WebBandit Disallow: / User-agent: EmailWolf Disallow: / User-agent: ExtractorPro Disallow: / User-agent: ia_archiver Disallow: / User-agent: Nikto Disallow: / User-agent: sqlmap Disallow: / User-agent: ZmEu Disallow: / User-agent: Havij Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebCopier Disallow: / User-agent: WebReaper Disallow: / User-agent: WebStripper Disallow: / User-agent: HTTrack Disallow: / User-agent: Teleport Disallow: / User-agent: Offline Explorer Disallow: / User-agent: spbot Disallow: / User-agent: Abonti Disallow: / User-agent: Nutch Disallow: / User-agent: Rank Tracker Disallow: / User-agent: OpenLinkProfiler Disallow: / User-agent: linkdexbot Disallow: / User-agent: Cliqzbot Disallow: / User-agent: admantx Disallow: / # --------------------------------------------------------------- # SECTION 6 — UNIVERSAL FALLBACK *** CRITICAL FIX *** # --------------------------------------------------------------- # THE BUG IN THE PREVIOUS FILE: # User-agent: * had ONLY "Crawl-delay: 2" — zero Allow rules. # Any crawler not named above (including the GSC Inspection # Tool) matched this block, saw no Allow rules, and had all # pages blocked by default — including /blog/ slugs. # # THE FIX: # User-agent: * now has full Allow rules for every public # page on the site. Only admin and backend paths are blocked. # --------------------------------------------------------------- User-agent: * Allow: /$ Allow: /home Allow: /events Allow: /events/ Allow: /event-view.php?id= Allow: /search_results$ Allow: /search-view.php?listing= Allow: /calender Allow: /weather Allow: /online-map Allow: /printable-map Allow: /trade_days_schedule Allow: /blog Allow: /blog/ Allow: /about-us Allow: /contact-us Allow: /new-vendor Allow: /sitemap.xml Allow: /sitemap-events.xml Allow: /sitemap-vendors.xml Allow: /sitemap-blog.xml Allow: /sitemap-static.xml Disallow: /cp/ Disallow: /admin/ Disallow: /join.php Disallow: /login/ Disallow: /login.php Disallow: /logout/ Disallow: /register/ Disallow: /account/ Disallow: /my-account/ Disallow: /password-reset/ Disallow: /forgot-password/ Allow: /blog-detail.php?view= Disallow: /blog-detail.php Disallow: /blog.php Disallow: /events.php Disallow: /calender.php Disallow: /search_results.php Disallow: /all_location.php Disallow: /ajaxactions.php Disallow: /ajaxcall.php Disallow: /index.php Disallow: /api/ Disallow: /ajax/ Disallow: /includes/ Disallow: /config/ Disallow: /cache/ Disallow: /tmp/ Disallow: /temp/ Disallow: /logs/ Disallow: /backup/ Disallow: /cron/ Disallow: /install/ Disallow: /cgi-bin/ Disallow: /cp/uploads/ Disallow: /*?session= Disallow: /*?utm_source= Disallow: /*?utm_medium= Disallow: /*?utm_campaign= Disallow: /*?fbclid= Disallow: /*?gclid= Disallow: /*?ref= Disallow: /*?affiliate= Disallow: /test/ Disallow: /staging/ Disallow: /dev/ Disallow: /beta/ Disallow: /old/ Crawl-delay: 2 # --------------------------------------------------------------- # SITEMAP DECLARATIONS # --------------------------------------------------------------- Sitemap: https://cantonshoppingguide.com/sitemap.xml Sitemap: https://cantonshoppingguide.com/sitemap-events.xml Sitemap: https://cantonshoppingguide.com/sitemap-vendors.xml Sitemap: https://cantonshoppingguide.com/sitemap-blog.xml Sitemap: https://cantonshoppingguide.com/sitemap-static.xml # ============================================================== # END OF robots.txt # # AFTER UPLOADING — verify at: # https://search.google.com/search-console/robots-testing-tool # # THEN request indexing for all blog URLs in GSC: # /blog/your-first-visit-to-canton-first-monday-trade-days... # /blog/a-day-at-canton-trade-days-from-coffee-stops... # /blog/how-can-a-flea-market-fulfill-all-your-shopping-needs # /blog/summer-trips-to-canton-first-monday-trade-days # # Last updated: May 21 2026 — Fixed blog-detail.php blocking # ==============================================================