Skip to content

Instantly share code, notes, and snippets.

@jroakes

jroakes/robots_secure_routes.json

Created October 14, 2021 12:48
Show Gist options
  • Save jroakes/bb5ac67ee4584f58f5a7714d62605da1 to your computer and use it in GitHub Desktop.
Save jroakes/bb5ac67ee4584f58f5a7714d62605da1 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
robots_secure_routes.json
{
"matched_disallows": {
"Googlebot": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Applebot": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Bingbot": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"msnbot": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Slurp": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Googlebot-Image": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"baiduspider": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Baiduspider": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"seznambot": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Teoma": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Yandex": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Yeti": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"msnbot-media": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Googlebot-News": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Googlebot-Video": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Googlebot-Mobile": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Mediapartners-Google": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"AdsBot-Google": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"DuckDuckBot": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"daumoa": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"OrangeBot": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"OrangeBot-Collector": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"StackRambler": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Twitterbot/1.0": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Mail.RU_Bot": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"naverbot": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"vebidoobot": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"MJ12bot": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"360Spider": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Sogou": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"rogerbot": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"deepcrawl": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"ByteSpider": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
],
"Neevabot": [
"/authwall",
"/learning*?auth=true",
"/learning*&auth=true",
"/learning/login*?redirect=",
"/learning/login*&redirect=",
"/oauth/v2/*",
"/oauth2/v2/*",
"/secure/",
"/uas/oauth/*",
"/uas/oauth2/*"
]
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment