# William Walker – www.williamwalker.de # Shopify Store · robots.txt (optimiert für SEO + AI-Sichtbarkeit) # Letzte Aktualisierung: 2026-03-08 # # Hinweis: Diese Datei muss als robots.txt.liquid in Shopify eingefügt werden. # Pfad: Online Store → Themes → Edit Code → Templates → robots.txt.liquid # ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ # ┃ Robots & Agent policy ┃ # ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ # ┃ Checkouts are for humans. ┃ # ┃ * Automated scraping, "buy-for-me" agents, or any end-to-end flow ┃ # ┃ that completes payment without a final human review step is not ┃ # ┃ permitted. ┃ # ┃ * Legitimate integrators must use the official Checkout Kit: ┃ # ┃ https://www.shopify.com/checkout-kit ┃ # ┃ ┃ # ┃ Terms of Service: https://www.shopify.com/legal/terms ┃ # ┃ Contact: bots@shopify.com ┃ # ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ # ═══════════════════════════════════════════════════════════════════════════ # ABSCHNITT 1: AI-RETRIEVAL-BOTS (Sichtbarkeit in AI-Suchergebnissen) # Diese Bots zeigen eure Produkte in ChatGPT, Perplexity, Claude etc. an. # Sie nutzen eure Inhalte NICHT zum Training, sondern nur zur Anzeige. # ═══════════════════════════════════════════════════════════════════════════ # --- OpenAI: ChatGPT Live-Browsing --- User-agent: ChatGPT-User Allow: /products/ Allow: /collections/ Allow: /pages/ Allow: /blogs/ Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /account Disallow: /search # --- OpenAI: SearchBot (AI-Suche) --- User-agent: OAI-SearchBot Allow: /products/ Allow: /collections/ Allow: /pages/ Allow: /blogs/ Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /account Disallow: /search # --- OpenAI: GPTBot (Training + Retrieval) --- User-agent: GPTBot Allow: /products/ Allow: /collections/ Allow: /pages/ Allow: /blogs/ Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /account Disallow: /search # --- Google AI (Extended) --- User-agent: Google-Extended Allow: /products/ Allow: /collections/ Allow: /pages/ Allow: /blogs/ Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /account Disallow: /search # --- Anthropic: Claude Retrieval --- User-agent: Claude-Web Allow: /products/ Allow: /collections/ Allow: /pages/ Allow: /blogs/ Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /account Disallow: /search # --- Anthropic: Claude SearchBot --- User-agent: Claude-SearchBot Allow: /products/ Allow: /collections/ Allow: /pages/ Allow: /blogs/ Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /account Disallow: /search # --- Anthropic: ClaudeBot --- User-agent: ClaudeBot Allow: /products/ Allow: /collections/ Allow: /pages/ Allow: /blogs/ Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /account Disallow: /search # --- Perplexity AI --- User-agent: PerplexityBot Allow: /products/ Allow: /collections/ Allow: /pages/ Allow: /blogs/ Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /account Disallow: /search # --- Microsoft Copilot / Bing Chat --- User-agent: Bingbot Allow: /products/ Allow: /collections/ Allow: /pages/ Allow: /blogs/ # --- Apple Siri / Spotlight AI --- User-agent: Applebot-Extended Allow: /products/ Allow: /collections/ Allow: /pages/ Allow: /blogs/ Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /account Disallow: /search # --- Meta AI --- User-agent: Meta-ExternalAgent Allow: /products/ Allow: /collections/ Allow: /pages/ Allow: /blogs/ Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /account Disallow: /search # --- Mistral AI --- User-agent: MistralAI-User Allow: /products/ Allow: /collections/ Allow: /pages/ Allow: /blogs/ Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /account Disallow: /search # ═══════════════════════════════════════════════════════════════════════════ # ABSCHNITT 2: AI-TRAINING-BOTS (BLOCKIERT) # Diese Bots sammeln Inhalte zum Trainieren von KI-Modellen. # Blockiert, damit eure Produkttexte und Bilder nicht in # Trainingsdaten landen. Entfernt die Disallow-Zeilen, wenn ihr # auch Training erlauben wollt. # ═══════════════════════════════════════════════════════════════════════════ # --- Common Crawl (breit genutzt für AI-Training) --- User-agent: CCBot Disallow: / # --- ByteDance / TikTok Training --- User-agent: Bytespider Disallow: / # --- Cohere AI Training --- User-agent: cohere-ai Disallow: / # --- Diffbot (AI Data Extraction) --- User-agent: Diffbot Disallow: / # --- Scrapy-basierte AI Crawler --- User-agent: Scrapy Disallow: / # ═══════════════════════════════════════════════════════════════════════════ # ABSCHNITT 3: STANDARD-SUCHMASCHINEN & ALLGEMEINE REGELN # (Euer bisheriges Setup, beibehalten) # ═══════════════════════════════════════════════════════════════════════════ User-agent: * Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /14090960953/checkouts Disallow: /14090960953/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /sf_private_access_tokens Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://www.williamwalker.de/sitemap.xml # ═══════════════════════════════════════════════════════════════════════════ # ABSCHNITT 4: SPEZIFISCHE BOTS (Euer bisheriges Setup) # ═══════════════════════════════════════════════════════════════════════════ # Google adsbot ignores robots.txt unless specifically named! User-agent: adsbot-google Disallow: /checkouts/ Disallow: /checkout Disallow: /carts Disallow: /orders Disallow: /14090960953/checkouts Disallow: /14090960953/orders Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /cdn/wpm/*.js Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /sf_private_access_tokens User-agent: Nutch Disallow: / User-agent: AhrefsBot Crawl-delay: 10 Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /14090960953/checkouts Disallow: /14090960953/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /sf_private_access_tokens Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://www.williamwalker.de/sitemap.xml User-agent: AhrefsSiteAudit Crawl-delay: 10 Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /14090960953/checkouts Disallow: /14090960953/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /sf_private_access_tokens Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://www.williamwalker.de/sitemap.xml User-agent: MJ12bot Crawl-delay: 10 User-agent: Pinterest Crawl-delay: 1