59375465ab4a12984be6927c87b7221a *DESCRIPTION
243cfc67b7b06ed97d9ffd2a858cc964 *LICENSE
e18179d70cce99a9fd478da38c2e98af *NAMESPACE
cafa2df7578031b2833191f6a63ce0f5 *NEWS.md
aaf15293fe21b392a1ad470dda79e7a0 *R/parse_robotstxt.R
61ee1fcd08e0b766aa68ebdefbefac2f *R/permissions.R
83dd8ac27bbb1f335732071808331fe5 *R/robotstxt.R
9b9ce4c2fe2fa265eb4a9c1dd422451e *R/tools.R
d6e9129e31a53131548114b7c17f60e9 *R/valid_robotstxt.R
ce05097740fc5813db89154783189022 *README.md
2cea5e45008aa4509378e335be999503 *build/vignette.rds
7f41bd4b25f00a3c21c43cd9909e0275 *inst/doc/using_robotstxt.R
8ea285b77b146385e74129e581eaff94 *inst/doc/using_robotstxt.Rmd
96179f8e0903c24fb9d0b7f9049896f9 *inst/doc/using_robotstxt.html
4d8921c3dc225aa07cd9b3f66cde3aff *inst/robotstxts/allow_single_bot.txt
e2b27c9301b4f24c3b2db3f8886b2dc1 *inst/robotstxts/crawl_delay.txt
8cb7a8b11cc42f1f77aa0670c43071a3 *inst/robotstxts/disallow_all_for_BadBot.txt
d35cb09d5662e686347b00b959c1550f *inst/robotstxts/disallow_all_for_all.txt
542398cc4e150fd288cea549bdca16a4 *inst/robotstxts/disallow_some_for_all.txt
9ca8636f08a496a82ed0a64694d005c8 *inst/robotstxts/disallow_two_at_once.txt
7215ee9c7d9dc229d2921a40e899ec5f *inst/robotstxts/empty.txt
d034a6e564902694ac6b372940786d80 *inst/robotstxts/host.txt
95d8d7433b4cd37b5630cf68500127f0 *inst/robotstxts/robots_amazon.txt
a09430767142d30f771f9726196048e1 *inst/robotstxts/robots_bundestag.txt
3b1a4b952779358ff7d6015b0dee0e68 *inst/robotstxts/robots_facebook.txt
c51eaab2c346b3a156c72097e52b10d9 *inst/robotstxts/robots_facebook_unsupported.txt
62dbaccb94d6ab8aa5e178339efbe127 *inst/robotstxts/robots_google.txt
658622dc6bda3645b94e37fa1ea38295 *inst/robotstxts/robots_new_york_times.txt
48a1701be03cf9590a8ef623c5f2e8e6 *inst/robotstxts/robots_pmeissner.txt
67dfdd8b9acd9363417c145388315dab *inst/robotstxts/robots_spiegel.txt
6ed07c3b696c3b4d47e1b3b4a42f5c7f *inst/robotstxts/robots_wikipedia.txt
b03c30fd553878fce698924ee1eca06e *inst/robotstxts/robots_wikipedia_20170706.txt
4637c3108489f0f0e5e283ef16884baf *inst/robotstxts/robots_yahoo.txt
6b8539213a6199601ee9d81c70a6ac31 *inst/robotstxts/selfhtml_Example.txt
2b9b53034510e3b66af8968145fe0a4f *inst/robotstxts/testing_comments.txt
ce5de2fe78b1401d341f3bef10998302 *man/get_robotstxt.Rd
dd31bc7e0488d099b4af153cf95a10f4 *man/get_robotstxt_http_get.Rd
25b7a8876d63a9c84ffc03a75cb6f0bc *man/guess_domain.Rd
cd93f324060d1d9b406539b788542663 *man/is_valid_robotstxt.Rd
714978c33b94323e62aeb3e279a98f20 *man/named_list.Rd
966b8100baa85950dacb4e7fb90b9381 *man/parse_robotstxt.Rd
9e73f5f9f42384288dca3943f95784d1 *man/path_allowed.Rd
2f020bbbed0611520962aaaf9612a78b *man/paths_allowed.Rd
d1fc25aefd1398fb3626b6606d7e8197 *man/print.robotstxt.Rd
05f83200d4748a76d213a47f2b734f7b *man/print.robotstxt_text.Rd
ead2016e814711c22df657c3990aa265 *man/remove_domain.Rd
aebc37e9fe758059e6e8aed151c6ebe3 *man/robotstxt.Rd
1c157ca6573579d2931f30cee53f9117 *man/rt_cache.Rd
87d75aa53d70fcb90171f9d565c78e5c *man/rt_get_comments.Rd
23be089e47639acd1e462e1e161e5d07 *man/rt_get_fields.Rd
c60729c79b9c093dc66e83127a537fc2 *man/rt_get_fields_worker.Rd
2ac3da4f9bbfd82068dc540e311f9cde *man/rt_get_rtxt.Rd
c665b068671d746af4236412b71197f6 *man/rt_get_useragent.Rd
aa83e831581c502e3d07a1dbc481ce11 *man/rt_list_rtxt.Rd
f7aed61031b0fc36693a9b3bf83823c8 *man/sanitize_path.Rd
95f9549dfb5fed889ea7c44448c8b194 *man/sanitize_permission_values.Rd
79cb2f7d72183f4418e5b5068c8bd9c9 *man/sanitize_permissions.Rd
480e44e1c814faf1886f057b8a375590 *tests/testthat.R
4f836dafef963308828a51e07c40582a *tests/testthat/test_parser.R
2128f04d4719818d858fe189e2e902e7 *tests/testthat/test_permissions.R
f1c2f190f8e683fb4784b864dd930bdc *tests/testthat/test_robotstxt.R
8ea285b77b146385e74129e581eaff94 *vignettes/using_robotstxt.Rmd
