|
1 # |
|
2 # robots.txt |
|
3 # |
|
4 # This file is to prevent the crawling and indexing of certain parts |
|
5 # of your site by web crawlers and spiders run by sites like Yahoo! |
|
6 # and Google. By telling these "robots" where not to go on your site, |
|
7 # you save bandwidth and server resources. |
|
8 # |
|
9 # This file will be ignored unless it is at the root of your host: |
|
10 # Used: http://example.com/robots.txt |
|
11 # Ignored: http://example.com/site/robots.txt |
|
12 # |
|
13 # For more information about the robots.txt standard, see: |
|
14 # http://www.robotstxt.org/robotstxt.html |
|
15 |
|
16 User-agent: * |
|
17 Crawl-delay: 10 |
|
18 # CSS, JS, Images |
|
19 Allow: /misc/*.css$ |
|
20 Allow: /misc/*.css? |
|
21 Allow: /misc/*.js$ |
|
22 Allow: /misc/*.js? |
|
23 Allow: /misc/*.gif |
|
24 Allow: /misc/*.jpg |
|
25 Allow: /misc/*.jpeg |
|
26 Allow: /misc/*.png |
|
27 Allow: /modules/*.css$ |
|
28 Allow: /modules/*.css? |
|
29 Allow: /modules/*.js$ |
|
30 Allow: /modules/*.js? |
|
31 Allow: /modules/*.gif |
|
32 Allow: /modules/*.jpg |
|
33 Allow: /modules/*.jpeg |
|
34 Allow: /modules/*.png |
|
35 Allow: /profiles/*.css$ |
|
36 Allow: /profiles/*.css? |
|
37 Allow: /profiles/*.js$ |
|
38 Allow: /profiles/*.js? |
|
39 Allow: /profiles/*.gif |
|
40 Allow: /profiles/*.jpg |
|
41 Allow: /profiles/*.jpeg |
|
42 Allow: /profiles/*.png |
|
43 Allow: /themes/*.css$ |
|
44 Allow: /themes/*.css? |
|
45 Allow: /themes/*.js$ |
|
46 Allow: /themes/*.js? |
|
47 Allow: /themes/*.gif |
|
48 Allow: /themes/*.jpg |
|
49 Allow: /themes/*.jpeg |
|
50 Allow: /themes/*.png |
|
51 # Directories |
|
52 Disallow: /includes/ |
|
53 Disallow: /misc/ |
|
54 Disallow: /modules/ |
|
55 Disallow: /profiles/ |
|
56 Disallow: /scripts/ |
|
57 Disallow: /themes/ |
|
58 # Files |
|
59 Disallow: /CHANGELOG.txt |
|
60 Disallow: /cron.php |
|
61 Disallow: /INSTALL.mysql.txt |
|
62 Disallow: /INSTALL.pgsql.txt |
|
63 Disallow: /INSTALL.sqlite.txt |
|
64 Disallow: /install.php |
|
65 Disallow: /INSTALL.txt |
|
66 Disallow: /LICENSE.txt |
|
67 Disallow: /MAINTAINERS.txt |
|
68 Disallow: /update.php |
|
69 Disallow: /UPGRADE.txt |
|
70 Disallow: /xmlrpc.php |
|
71 # Paths (clean URLs) |
|
72 Disallow: /admin/ |
|
73 Disallow: /comment/reply/ |
|
74 Disallow: /filter/tips/ |
|
75 Disallow: /node/add/ |
|
76 Disallow: /search/ |
|
77 Disallow: /user/register/ |
|
78 Disallow: /user/password/ |
|
79 Disallow: /user/login/ |
|
80 Disallow: /user/logout/ |
|
81 # Paths (no clean URLs) |
|
82 Disallow: /?q=admin/ |
|
83 Disallow: /?q=comment/reply/ |
|
84 Disallow: /?q=filter/tips/ |
|
85 Disallow: /?q=node/add/ |
|
86 Disallow: /?q=search/ |
|
87 Disallow: /?q=user/password/ |
|
88 Disallow: /?q=user/register/ |
|
89 Disallow: /?q=user/login/ |
|
90 Disallow: /?q=user/logout/ |