| author | ymh <ymh.work@gmail.com> |
| Thu, 16 Nov 2017 16:31:09 +0100 | |
| changeset 554 | f28a539ba106 |
| parent 541 | e756a8c72c3d |
| permissions | -rwxr-xr-x |
|
541
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
1 |
# |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
2 |
# robots.txt |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
3 |
# |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
4 |
# This file is to prevent the crawling and indexing of certain parts |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
5 |
# of your site by web crawlers and spiders run by sites like Yahoo! |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
6 |
# and Google. By telling these "robots" where not to go on your site, |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
7 |
# you save bandwidth and server resources. |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
8 |
# |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
9 |
# This file will be ignored unless it is at the root of your host: |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
10 |
# Used: http://example.com/robots.txt |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
11 |
# Ignored: http://example.com/site/robots.txt |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
12 |
# |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
13 |
# For more information about the robots.txt standard, see: |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
14 |
# http://www.robotstxt.org/robotstxt.html |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
15 |
|
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
16 |
User-agent: * |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
17 |
Crawl-delay: 10 |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
18 |
# CSS, JS, Images |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
19 |
Allow: /misc/*.css$ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
20 |
Allow: /misc/*.css? |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
21 |
Allow: /misc/*.js$ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
22 |
Allow: /misc/*.js? |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
23 |
Allow: /misc/*.gif |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
24 |
Allow: /misc/*.jpg |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
25 |
Allow: /misc/*.jpeg |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
26 |
Allow: /misc/*.png |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
27 |
Allow: /modules/*.css$ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
28 |
Allow: /modules/*.css? |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
29 |
Allow: /modules/*.js$ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
30 |
Allow: /modules/*.js? |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
31 |
Allow: /modules/*.gif |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
32 |
Allow: /modules/*.jpg |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
33 |
Allow: /modules/*.jpeg |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
34 |
Allow: /modules/*.png |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
35 |
Allow: /profiles/*.css$ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
36 |
Allow: /profiles/*.css? |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
37 |
Allow: /profiles/*.js$ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
38 |
Allow: /profiles/*.js? |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
39 |
Allow: /profiles/*.gif |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
40 |
Allow: /profiles/*.jpg |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
41 |
Allow: /profiles/*.jpeg |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
42 |
Allow: /profiles/*.png |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
43 |
Allow: /themes/*.css$ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
44 |
Allow: /themes/*.css? |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
45 |
Allow: /themes/*.js$ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
46 |
Allow: /themes/*.js? |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
47 |
Allow: /themes/*.gif |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
48 |
Allow: /themes/*.jpg |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
49 |
Allow: /themes/*.jpeg |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
50 |
Allow: /themes/*.png |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
51 |
# Directories |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
52 |
Disallow: /includes/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
53 |
Disallow: /misc/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
54 |
Disallow: /modules/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
55 |
Disallow: /profiles/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
56 |
Disallow: /scripts/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
57 |
Disallow: /themes/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
58 |
# Files |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
59 |
Disallow: /CHANGELOG.txt |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
60 |
Disallow: /cron.php |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
61 |
Disallow: /INSTALL.mysql.txt |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
62 |
Disallow: /INSTALL.pgsql.txt |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
63 |
Disallow: /INSTALL.sqlite.txt |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
64 |
Disallow: /install.php |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
65 |
Disallow: /INSTALL.txt |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
66 |
Disallow: /LICENSE.txt |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
67 |
Disallow: /MAINTAINERS.txt |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
68 |
Disallow: /update.php |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
69 |
Disallow: /UPGRADE.txt |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
70 |
Disallow: /xmlrpc.php |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
71 |
# Paths (clean URLs) |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
72 |
Disallow: /admin/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
73 |
Disallow: /comment/reply/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
74 |
Disallow: /filter/tips/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
75 |
Disallow: /node/add/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
76 |
Disallow: /search/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
77 |
Disallow: /user/register/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
78 |
Disallow: /user/password/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
79 |
Disallow: /user/login/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
80 |
Disallow: /user/logout/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
81 |
# Paths (no clean URLs) |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
82 |
Disallow: /?q=admin/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
83 |
Disallow: /?q=comment/reply/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
84 |
Disallow: /?q=filter/tips/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
85 |
Disallow: /?q=node/add/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
86 |
Disallow: /?q=search/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
87 |
Disallow: /?q=user/password/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
88 |
Disallow: /?q=user/register/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
89 |
Disallow: /?q=user/login/ |
|
e756a8c72c3d
integrate drupal and correct build process. update version
ymh <ymh.work@gmail.com>
parents:
diff
changeset
|
90 |
Disallow: /?q=user/logout/ |