User-agent: * # Block directories Disallow: /AjaxFeeds/ Disallow: /App_Code/ Disallow: /Bin/ Disallow: /controls/ Disallow: /css/ Disallow: /csv/ Disallow: /ErrorPages/ Disallow: /feeds/ Disallow: /flash/ Disallow: /imgs/ Disallow: /js/ Disallow: /Login/ Disallow: /Monitor/ Disallow: /PageControls/ Disallow: /TextSources/ Disallow: /utp/ Disallow: /xml/ # Block specific pages Disallow: /HistoryHolder1.html Disallow: /newsletter-error.aspx Disallow: /newsletter-example.aspx Disallow: /*?Accommodatie/Bekijken Disallow: /LinkPopup.aspx Disallow: /LinkPopup.aspx* Disallow: /LinkPopup.aspx?* Disallow: /linkPopup.aspx Disallow: /linkPopup.aspx* Disallow: /linkPopup.aspx?* Disallow: /linkpopup.aspx Disallow: /linkpopup.aspx* Disallow: /linkpopup.aspx?* Disallow: /LinkPopup2.aspx Disallow: /LinkPopup2.aspx* Disallow: /LinkPopup2.aspx?* Disallow: /linkPopup2.aspx Disallow: /linkPopup2.aspx* Disallow: /linkPopup2.aspx?* Disallow: /linkpopup2.aspx Disallow: /linkpopup2.aspx* Disallow: /linkpopup2.aspx?* # Additional Settings Crawl-delay: 1 # 1 second between requests Request-rate: 1/1 # 1 page per 1 seconds