# Got this from thinkhost.com & then highly modified
#this is a master robots text -- modify for each site
#first part is from Drupal 5.2 version to use on all sites

# $Id: robots.txt,v 1.7.2.1 2007/03/23 18:57:07 drumm Exp $
#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/wc/robots.html
#
# For syntax checking, see:
# http://www.sxw.org.uk/computing/robots/check.html

User-agent: *
Crawl-delay: 10
# Directories
Disallow: /database/
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /sites/
Disallow: /themes/
Disallow: /scripts/
Disallow: /updates/
Disallow: /profiles/
# Files
Disallow: /xmlrpc.php
Disallow: /cron.php
Disallow: /update.php
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /CHANGELOG.txt
Disallow: /MAINTAINERS.txt
Disallow: /LICENSE.txt
Disallow: /UPGRADE.txt
# Paths (clean URLs)
Disallow: /admin/
Disallow: /aggregator/
Disallow: /comment/reply/
Disallow: /contact/
Disallow: /logout/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=aggregator/
Disallow: /?q=comment/reply/
Disallow: /?q=contact/
Disallow: /?q=logout/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/

# SITE SPECIFIC
# CAPNJ
Disallow: /drupal/capnj/albums/
Disallow: /drupal/capnj/images/
Disallow: /drupal/capnj/tmp/

# SITE SPECIFIC
#dajudge/CRM
Disallow: /civicrm/

# SITE SPECIFIC
#MINE SITE - TOTAL EXCLUSION - BOTH PERSONAL AND COURT
Disallow: /forum/mine/
Disallow: /drupal/mine/
# CAPNJ
Disallow: /drupal/capnj/albums/
Disallow: /drupal/capnj/files/
Disallow: /drupal/capnj/images/

# END OF THE DRUPAL AND CALENDAR SITES

#ADDED PER http://www.robotstxt.org/wc/active/html/contact.html
User-agent:	Alexibot
User-agent:	Aqua_Products
User-agent:	asterias
User-agent:	b2w/0.1
User-agent:	BackDoorBot/1.0
User-agent:	BlowFish/1.0
User-agent:	Bookmark search tool
User-agent:	BotALot
User-agent:	BotRightHere
User-agent:	BuiltBotTough
User-agent:	Bullseye/1.0
User-agent:	BunnySlippers
User-agent:	CheeseBot
User-agent:	CherryPicker
User-agent:	CherryPickerElite/1.0
User-agent:	CherryPickerSE/1.0
User-agent:	Copernic
User-agent:	CopyRightCheck
User-agent:	cosmos
User-agent:	Crescent
User-agent:	Crescent Internet ToolPak HTTP OLE Control v.1.0
User-agent:	DittoSpyder
User-agent:	EmailCollector
User-agent:	EmailSiphon
User-agent:	EmailWolf
User-agent:	EroCrawler
User-agent:	ExtractorPro
User-agent:	FairAd Client
User-agent:	Feedster Crawler/1.0
User-agent:	Flaming AttackBot
User-agent:	Foobot
User-agent:	fusionbot
User-agent:	Gaisbot
User-agent:	GetRight/4.2
User-agent:	Harvest/1.5
User-agent:	hloader
User-agent:	httplib
User-agent:	HTTrack 3.0
User-agent:	humanlinks
User-agent:	InfoNaviRobot
User-agent:	Iron33/1.0.2
User-agent:	JennyBot
User-agent:	Kenjin Spider
User-agent:	Keyword Density/0.9
User-agent:	larbin
User-agent:	LexiBot
User-agent:	libWeb/clsHTTP
User-agent:	LinkextractorPro
User-agent:	LinkScan/8.1a Unix
User-agent:	LinkWalker
User-agent:	LNSpiderguy
User-agent:	lwp-trivial
User-agent:	lwp-trivial/1.34
User-agent:	Mata Hari
User-agent:	Microsoft URL Control
User-agent:	Microsoft URL Control - 5.01.4511
User-agent:	Microsoft URL Control - 6.00.8169
User-agent:	MIIxpc
User-agent:	MIIxpc/4.2
User-agent:	Mister PiX
User-agent:	moget
User-agent:	moget/2.1
User-agent:	Mozilla/4.0 (compatible; BullsEye; Windows 95)
User-agent:	MSIECrawler
User-agent:	NetAnts
User-agent:	NetMechanic
User-agent:	NICErsPRO
User-agent:	Offline Explorer
User-agent:	Openbot
User-agent:	Openfind
User-agent:	Openfind data gatherer
User-agent:	Oracle Ultra Search
User-agent:	PerMan
User-agent:	ProPowerBot/2.14
User-agent:	ProWebWalker
User-agent:	psbot
User-agent:	Python-urllib
User-agent:	QueryN Metasearch
User-agent:	Radiation Retriever 1.1
User-agent:	RepoMonkey
User-agent:	RepoMonkey Bait & Tackle/v1.01
User-agent:	RMA
User-agent:	searchpreview
User-agent:	SiteSnagger
User-agent:	smallbear
User-agent:	SpankBot
User-agent:	spanner
User-agent:	suzuran
User-agent:	Szukacz/1.4
User-agent:	Technoratibot/0.6
User-agent:	Teleport
User-agent:	TeleportPro
User-agent:	Telesoft
User-agent:	The Intraformant
User-agent:	TheNomad
User-agent:	TightTwatBot
User-agent:	toCrawl/UrlDispatcher
User-agent:	True_Robot
User-agent:	True_Robot/1.0
User-agent:	turingos
User-agent:	TurnitinBot
User-agent:	TurnitinBot/1.5
User-agent:	URL Control
User-agent:	URL_Spider_Pro
User-agent:	URLy Warning
User-agent:	VCI
User-agent:	VCI WebViewer VCI WebViewer Win32
User-agent:	Web Image Collector
User-agent:	WebAuto
User-agent:	WebBandit
User-agent:	WebBandit/3.50
User-agent:	WebCapture 2.0
User-agent:	WebCopier
User-agent:	WebCopier v.2.2
User-agent:	WebCopier v3.2a
User-agent:	WebEnhancer
User-agent:	WebSauger
User-agent:	Website Quester
User-agent:	Webster Pro
User-agent:	WebStripper
User-agent:	WebZip
User-agent:	WebZip/4.0
User-agent:	WebZIP/4.21
User-agent:	WebZIP/5.0
User-agent:	Wget
User-agent:	wget
User-agent:	Wget/1.5.3
User-agent:	Wget/1.6
User-agent:	WWW-Collector-E
User-agent:	Xenu's
User-agent:	Xenu's Link Sleuth 1.1c
User-agent:	Zeus
User-agent:	Zeus 32297 Webster Pro V2.9 Win32
User-agent:	Zeus Link Scout
Disallow: /