# robots.txt for http://www.mathworks.com and subdomains
# Please do not update this file without contacting the owner
# Owner: webops at mathworks.com
# Note 1 for updating: Please keep list alphabetized by URL.
# Note 2 for updating: When making an update, it needs to be updated for /de/, /fr/, /en/ sections as well.

User-agent: *
Disallow: *_ja_JP.html*
Disallow: *_ko_KR.html*
Disallow: *_zh_CN.html*
Disallow: */shared-content/*
Disallow: /_site_doc
Disallow: /add-ons/
Disallow: /archive/
Disallow: /brandguide/
Disallow: /campaigns/portals/
Disallow: /campaigns/products/trials/forms/
Disallow: /cgi-bin
Disallow: /company/events/mab
Disallow: /company/events/research-summit
Disallow: /content/dam/mathworks/ebook/gated/
Disallow: /content/dam/mathworks/mathworks-dot-com/cmsimages/connections/gated/
Disallow: /content/dam/mathworks/mathworks-dot-com/images/company/jobs
Disallow: /content/dam/mathworks/price-lists/
Disallow: /content/dam/mathworks/white-paper/gated/
Disallow: /content/mathworks/
Disallow: /develop/
Allow: /develop/review/
Disallow: /guides/
Disallow: /help/deeplearning/index.html;*
Disallow: /help/pdf_doc/
Disallow: /help/releases/
Disallow: /license/
Disallow: /licensecenter/classroom/
Disallow: /matlabcentral/answers/?
Allow: /matlabcentral/answers/?page=*
Disallow: /matlabcentral/answers/?page=*&*
Allow: /matlabcentral/answers/?page=*&s_tid=gn_mlc_ans_ans
Allow: /matlabcentral/answers/?s_tid=gn_mlc_ans&page=*
Allow: /matlabcentral/answers/?s_tid=gn_mlc_ans_ans
Disallow: /matlabcentral/answers/?s_tid=gn_mlc_ans_ans&*
Disallow: /matlabcentral/answers/?s_tid=gn_mlc_ans_ans&*page=*&*
Disallow: /matlabcentral/answers/uploaded_files/*
Disallow: /matlabcentral/answers/questions*
Disallow: /matlabcentral/cody/activities
Disallow: /matlabcentral/cody/players/*
Disallow: /matlabcentral/cody/problems/*/players
Disallow: /matlabcentral/cody/problems/*/solutions
Disallow: /matlabcentral/cody/problems/*/solutions/*
Disallow: /matlabcentral/cody/solutions
Disallow: /matlabcentral/fileexchange/*?*download=true*
Disallow: /matlabcentral/fileexchange/?
Allow: /matlabcentral/fileexchange/?term=type:app
Allow: /matlabcentral/fileexchange/?term=type:app&page=*
Disallow: /matlabcentral/fileexchange/?term=type:app*&*
Allow: /matlabcentral/fileexchange/?term=type:class
Allow: /matlabcentral/fileexchange/?term=type:class&page=*
Disallow: /matlabcentral/fileexchange/?term=type:class*&*
Allow: /matlabcentral/fileexchange/?term=type:driver
Allow: /matlabcentral/fileexchange/?term=type:driver&page=*
Disallow: /matlabcentral/fileexchange/?term=type:driver*&*
Allow: /matlabcentral/fileexchange/?term=type:example
Allow: /matlabcentral/fileexchange/?term=type:example&page=*
Disallow: /matlabcentral/fileexchange/?term=type:example*&*
Allow: /matlabcentral/fileexchange/?term=type:function
Allow: /matlabcentral/fileexchange/?term=type:function&page=*
Disallow: /matlabcentral/fileexchange/?term=type:function*&*
Allow: /matlabcentral/fileexchange/?term=type:model
Allow: /matlabcentral/fileexchange/?term=type:model&page=*
Disallow: /matlabcentral/fileexchange/?term=type:model*&*
Allow: /matlabcentral/fileexchange/?term=type:script
Allow: /matlabcentral/fileexchange/?term=type:script&page=*
Disallow: /matlabcentral/fileexchange/?term=type:script*&*
Allow: /matlabcentral/fileexchange/?term=type:video
Allow: /matlabcentral/fileexchange/?term=type:video&page=*
Disallow: /matlabcentral/fileexchange/?term=type:video*&*
Disallow: /matlabcentral/fileexchange/authors/?
Allow: /matlabcentral/fileexchange/authors/?page=*
Disallow: /matlabcentral/fileexchange/authors/?page=*&*
Disallow: /matlabcentral/fileexchange/authors/contact/
Disallow: /matlabcentral/fileexchange/download.do
Disallow: /matlabcentral/fileexchange/feed.atom
Disallow: /matlabcentral/fileexchange/feedbacks
Disallow: /matlabcentral/fileexchange/index
Disallow: /matlabcentral/fileexchange/tags
Disallow: /matlabcentral/mlc-downloads/
Disallow: /matlabcentral/profile/api/v1/active/*
Disallow: /matlabcentral/profile/contact/*
Disallow: /moler/ncm-download.html
Disallow: /products/connections/portal/
Disallow: /products/connections/trials
Disallow: /quality/
Disallow: /searchresults*
Disallow: /searchresults/
Allow: /searchresults/*videos_all*videos_jp*facets
Disallow: /store/link/
Disallow: /support/bugreports/
Disallow: /support/pricing/
Disallow: /tagteam
Disallow: /test/
Disallow: /videos/*.feedback.html
Disallow: /videos/video-feedback.html
Disallow: /videos/video-feedback-confirmation.html
Disallow: /web_downloads


User-agent: Baiduspider
Disallow: /academia/books/search.html
Disallow: /company/user_stories/search.html
Disallow: /company/newsletters/search.html
Disallow: /hardware-support.html
Disallow: /hardware-support/instrument-control-toolbox/drivers-search.html
Disallow: /products/connections/search-products.html
Disallow: /products/connections/search-services.html
Disallow: /search.html
Disallow: /support/search.html
Disallow: /videos/search.html
Disallow: /searchresults*
Disallow: /searchresults/


User-agent: Slurp
Disallow: /company/aboutus/policies_statements/travel.html
Crawl-delay: 60


# /de/ below


User-agent: *
Disallow: /de/*/shared-content/*
Disallow: /de/_site_doc
Disallow: /de/add-ons/
Disallow: /de/archive/
Disallow: /de/brandguide/
Disallow: /de/campaigns/portals/
Disallow: /de/campaigns/products/trials/forms/
Disallow: /de/cgi-bin
Disallow: /de/company/events/mab/
Disallow: /de/company/events/research-summit/
Disallow: /de/content/dam/mathworks/ebook/gated/
Disallow: /de/content/dam/mathworks/mathworks-dot-com/cmsimages/connections/gated/
Disallow: /de/content/dam/mathworks/mathworks-dot-com/images/company/jobs
Disallow: /de/content/dam/mathworks/price-lists/
Disallow: /de/content/dam/mathworks/white-paper/gated/
Disallow: /de/content/mathworks
Disallow: /de/develop/
Allow: /de/develop/review/
Disallow: /de/guides/
Disallow: /de/help/deeplearning/index.html;*
Disallow: /de/help/pdf_doc/
Disallow: /de/help/releases/
Disallow: /de/licensecenter/classroom/
Disallow: /de/matlabcentral/answers/?
Allow: /de/matlabcentral/answers/?page=*
Disallow: /de/matlabcentral/answers/?page=*&*
Allow: /de/matlabcentral/answers/?page=*&s_tid=gn_mlc_ans_ans
Allow: /de/matlabcentral/answers/?s_tid=gn_mlc_ans&page=*
Allow: /de/matlabcentral/answers/?s_tid=gn_mlc_ans_ans
Disallow: /de/matlabcentral/answers/?s_tid=gn_mlc_ans_ans&*
Disallow: /de/matlabcentral/answers/?s_tid=gn_mlc_ans_ans&*page=*&*
Disallow: /de/matlabcentral/answers/uploaded_files/*
Disallow: /de/matlabcentral/answers/questions*
Disallow: /de/matlabcentral/cody/activities
Disallow: /de/matlabcentral/cody/players/*
Disallow: /de/matlabcentral/cody/problems/*/players
Disallow: /de/matlabcentral/cody/problems/*/solutions
Disallow: /de/matlabcentral/cody/problems/*/solutions/*
Disallow: /de/matlabcentral/cody/solutions
Disallow: /de/matlabcentral/fileexchange/*?*download=true*
Disallow: /de/matlabcentral/fileexchange/?
Allow: /de/matlabcentral/fileexchange/?term=type:app
Allow: /de/matlabcentral/fileexchange/?term=type:app&page=*
Disallow: /de/matlabcentral/fileexchange/?term=type:app*&*
Allow: /de/matlabcentral/fileexchange/?term=type:class
Allow: /de/matlabcentral/fileexchange/?term=type:class&page=*
Disallow: /de/matlabcentral/fileexchange/?term=type:class*&*
Allow: /de/matlabcentral/fileexchange/?term=type:driver
Allow: /de/matlabcentral/fileexchange/?term=type:driver&page=*
Disallow: /de/matlabcentral/fileexchange/?term=type:driver*&*
Allow: /de/matlabcentral/fileexchange/?term=type:example
Allow: /de/matlabcentral/fileexchange/?term=type:example&page=*
Disallow: /de/matlabcentral/fileexchange/?term=type:example*&*
Allow: /de/matlabcentral/fileexchange/?term=type:function
Allow: /de/matlabcentral/fileexchange/?term=type:function&page=*
Disallow: /de/matlabcentral/fileexchange/?term=type:function*&*
Allow: /de/matlabcentral/fileexchange/?term=type:model
Allow: /de/matlabcentral/fileexchange/?term=type:model&page=*
Disallow: /de/matlabcentral/fileexchange/?term=type:model*&*
Allow: /de/matlabcentral/fileexchange/?term=type:script
Allow: /de/matlabcentral/fileexchange/?term=type:script&page=*
Disallow: /de/matlabcentral/fileexchange/?term=type:script*&*
Allow: /de/matlabcentral/fileexchange/?term=type:video
Allow: /de/matlabcentral/fileexchange/?term=type:video&page=*
Disallow: /de/matlabcentral/fileexchange/?term=type:video*&*
Disallow: /de/matlabcentral/fileexchange/authors/?
Allow: /de/matlabcentral/fileexchange/authors/?page=*
Disallow: /de/matlabcentral/fileexchange/authors/?page=*&*
Disallow: /de/matlabcentral/fileexchange/authors/contact/
Disallow: /de/matlabcentral/fileexchange/download.do
Disallow: /de/matlabcentral/fileexchange/feed.atom
Disallow: /de/matlabcentral/fileexchange/feedbacks
Disallow: /de/matlabcentral/fileexchange/index
Disallow: /de/matlabcentral/fileexchange/tags
Disallow: /de/matlabcentral/mlc-downloads/
Disallow: /de/matlabcentral/profile/api/v1/active/*
Disallow: /de/matlabcentral/profile/contact/*
Disallow: /de/moler/ncm-download.html
Disallow: /de/products/connections/portal/
Disallow: /de/products/connections/trials
Disallow: /de/quality/
Disallow: /de/searchresults*
Disallow: /de/searchresults/
Allow: /de/searchresults/*videos_all*videos_jp*facets
Disallow: /de/store/link/
Disallow: /de/support/bugreports/
Disallow: /de/support/pricing/
Disallow: /de/tagteam
Disallow: /de/test/
Disallow: /de/videos/*.feedback.html
Disallow: /de/videos/video-feedback.html
Disallow: /de/videos/video-feedback-confirmation.html
Disallow: /de/web_downloads


User-agent: Baiduspider
Disallow: /de/academia/books/search.html
Disallow: /de/company/user_stories/search.html
Disallow: /de/company/newsletters/search.html
Disallow: /de/hardware-support.html
Disallow: /de/hardware-support/instrument-control-toolbox/drivers-search.html
Disallow: /de/products/connections/search-products.html
Disallow: /de/products/connections/search-services.html
Disallow: /de/search.html
Disallow: /de/support/search.html
Disallow: /de/videos/search.html
Disallow: /de/searchresults*
Disallow: /de/searchresults/


User-agent: Slurp
Disallow: /de/company/aboutus/policies_statements/travel.html
Crawl-delay: 60


# /fr/ below


User-agent: *
Disallow: /fr/*/shared-content/*
Disallow:  /fr/_site_doc
Disallow: /fr/add-ons/
Disallow: /fr/archive/
Disallow: /fr/brandguide/
Disallow: /fr/campaigns/portals/
Disallow: /fr/campaigns/products/trials/forms/
Disallow: /fr/cgi-bin
Disallow: /fr/company/events/mab/
Disallow: /fr/company/events/research-summit/
Disallow: /fr/content/dam/mathworks/ebook/gated/
Disallow: /fr/content/dam/mathworks/mathworks-dot-com/cmsimages/connections/gated/
Disallow: /fr/content/dam/mathworks/mathworks-dot-com/images/company/jobs
Disallow: /fr/content/dam/mathworks/price-lists/
Disallow: /fr/content/dam/mathworks/white-paper/gated/
Disallow: /fr/content/mathworks/
Disallow: /fr/develop/
Allow: /fr/develop/review/
Disallow: /fr/guides/
Disallow: /fr/help/deeplearning/index.html;*
Disallow: /fr/help/pdf_doc/
Disallow: /fr/help/releases/
Disallow: /fr/licensecenter/classroom/
Disallow: /fr/matlabcentral/answers/?
Allow: /fr/matlabcentral/answers/?page=*
Disallow: /fr/matlabcentral/answers/?page=*&*
Allow: /fr/matlabcentral/answers/?page=*&s_tid=gn_mlc_ans_ans
Allow: /fr/matlabcentral/answers/?s_tid=gn_mlc_ans&page=*
Allow: /fr/matlabcentral/answers/?s_tid=gn_mlc_ans_ans
Disallow: /fr/matlabcentral/answers/?s_tid=gn_mlc_ans_ans&*
Disallow: /fr/matlabcentral/answers/?s_tid=gn_mlc_ans_ans&*page=*&*
Disallow: /fr/matlabcentral/answers/uploaded_files/*
Disallow: /fr/matlabcentral/answers/questions*
Disallow: /fr/matlabcentral/cody/activities
Disallow: /fr/matlabcentral/cody/players/*
Disallow: /fr/matlabcentral/cody/problems/*/players
Disallow: /fr/matlabcentral/cody/problems/*/solutions
Disallow: /fr/matlabcentral/cody/problems/*/solutions/*
Disallow: /fr/matlabcentral/cody/solutions
Disallow: /fr/matlabcentral/fileexchange/*?*download=true*
Disallow: /fr/matlabcentral/fileexchange/?
Allow: /fr/matlabcentral/fileexchange/?term=type:app
Allow: /fr/matlabcentral/fileexchange/?term=type:app&page=*
Disallow: /fr/matlabcentral/fileexchange/?term=type:app*&*
Allow: /fr/matlabcentral/fileexchange/?term=type:class
Allow: /fr/matlabcentral/fileexchange/?term=type:class&page=*
Disallow: /fr/matlabcentral/fileexchange/?term=type:class*&*
Allow: /fr/matlabcentral/fileexchange/?term=type:driver
Allow: /fr/matlabcentral/fileexchange/?term=type:driver&page=*
Disallow: /fr/matlabcentral/fileexchange/?term=type:driver*&*
Allow: /fr/matlabcentral/fileexchange/?term=type:example
Allow: /fr/matlabcentral/fileexchange/?term=type:example&page=*
Disallow: /fr/matlabcentral/fileexchange/?term=type:example*&*
Allow: /fr/matlabcentral/fileexchange/?term=type:function
Allow: /fr/matlabcentral/fileexchange/?term=type:function&page=*
Disallow: /fr/matlabcentral/fileexchange/?term=type:function*&*
Allow: /fr/matlabcentral/fileexchange/?term=type:model
Allow: /fr/matlabcentral/fileexchange/?term=type:model&page=*
Disallow: /fr/matlabcentral/fileexchange/?term=type:model*&*
Allow: /fr/matlabcentral/fileexchange/?term=type:script
Allow: /fr/matlabcentral/fileexchange/?term=type:script&page=*
Disallow: /fr/matlabcentral/fileexchange/?term=type:script*&*
Allow: /fr/matlabcentral/fileexchange/?term=type:video
Allow: /fr/matlabcentral/fileexchange/?term=type:video&page=*
Disallow: /fr/matlabcentral/fileexchange/?term=type:video*&*
Disallow: /fr/matlabcentral/fileexchange/authors/?
Allow: /fr/matlabcentral/fileexchange/authors/?page=*
Disallow: /fr/matlabcentral/fileexchange/authors/?page=*&*
Disallow: /fr/matlabcentral/fileexchange/authors/contact/
Disallow: /fr/matlabcentral/fileexchange/download.do
Disallow: /fr/matlabcentral/fileexchange/feed.atom
Disallow: /fr/matlabcentral/fileexchange/feedbacks
Disallow: /fr/matlabcentral/fileexchange/index
Disallow: /fr/matlabcentral/fileexchange/tags
Disallow: /fr/matlabcentral/mlc-downloads/
Disallow: /fr/matlabcentral/profile/api/v1/active/*
Disallow: /fr/matlabcentral/profile/contact/*
Disallow: /fr/moler/ncm-download.html
Disallow: /fr/products/connections/portal/
Disallow: /fr/products/connections/trials
Disallow: /fr/quality/
Disallow: /fr/searchresults*
Disallow: /fr/searchresults/
Allow: /fr/searchresults/*videos_all*videos_jp*facets
Disallow: /fr/store/link/
Disallow: /fr/support/bugreports/
Disallow: /fr/support/pricing/
Disallow: /fr/tagteam
Disallow: /fr/test/
Disallow: /fr/videos/*.feedback.html
Disallow: /fr/videos/video-feedback.html
Disallow: /fr/videos/video-feedback-confirmation.html
Disallow: /fr/web_downloads


User-agent: Baiduspider
Disallow: /fr/academia/books/search.html
Disallow: /fr/company/user_stories/search.html
Disallow: /fr/company/newsletters/search.html
Disallow: /fr/hardware-support.html
Disallow: /fr/hardware-support/instrument-control-toolbox/drivers-search.html
Disallow: /fr/products/connections/search-products.html
Disallow: /fr/products/connections/search-services.html
Disallow: /fr/search.html
Disallow: /fr/support/search.html
Disallow: /fr/videos/search.html
Disallow: /fr/searchresults*
Disallow: /fr/searchresults/


User-agent: Slurp
Disallow: /fr/company/aboutus/policies_statements/travel.html
Crawl-delay: 60


# /en/ below


User-agent: *
Disallow: /en/*_ja_JP.html*
Disallow: /en/*_ko_KR.html*
Disallow: /en/*_zh_CN.html*
Disallow: /en/*/shared-content/*
Disallow: /en/_site_doc
Disallow: /en/add-ons/
Disallow: /en/archive/
Disallow: /en/brandguide/
Disallow: /en/campaigns/portals/
Disallow: /en/campaigns/products/trials/forms/
Disallow: /en/cgi-bin
Disallow: /en/company/events/mab/
Disallow: /en/company/events/research-summit/
Disallow: /en/content/dam/mathworks/ebook/gated/
Disallow: /en/content/dam/mathworks/mathworks-dot-com/cmsimages/connections/gated/
Disallow: /en/content/dam/mathworks/mathworks-dot-com/images/company/jobs
Disallow: /en/content/dam/mathworks/price-lists/
Disallow: /en/content/dam/mathworks/white-paper/gated/
Disallow: /en/content/mathworks/
Disallow: /en/develop/
Allow: /en/develop/review/
Disallow: /en/guides/
Disallow: /en/help/deeplearning/index.html;*
Disallow: /en/help/pdf_doc/
Disallow: /en/help/releases/
Disallow: /en/licensecenter/classroom/
Disallow: /en/matlabcentral/answers/?
Allow: /en/matlabcentral/answers/?page=*
Disallow: /en/matlabcentral/answers/?page=*&*
Allow: /en/matlabcentral/answers/?page=*&s_tid=gn_mlc_ans_ans
Allow: /en/matlabcentral/answers/?s_tid=gn_mlc_ans&page=*
Allow: /en/matlabcentral/answers/?s_tid=gn_mlc_ans_ans
Disallow: /en/matlabcentral/answers/?s_tid=gn_mlc_ans_ans&*
Disallow: /en/matlabcentral/answers/?s_tid=gn_mlc_ans_ans&*page=*&*
Disallow: /en/matlabcentral/answers/uploaded_files/*
Disallow: /en/matlabcentral/answers/questions*
Disallow: /en/matlabcentral/cody/activities
Disallow: /en/matlabcentral/cody/players/*
Disallow: /en/matlabcentral/cody/problems/*/players
Disallow: /en/matlabcentral/cody/problems/*/solutions
Disallow: /en/matlabcentral/cody/problems/*/solutions/*
Disallow: /en/matlabcentral/cody/solutions
Disallow: /en/matlabcentral/fileexchange/*?*download=true*
Disallow: /en/matlabcentral/fileexchange/?
Allow: /en/matlabcentral/fileexchange/?term=type:app
Allow: /en/matlabcentral/fileexchange/?term=type:app&page=*
Disallow: /en/matlabcentral/fileexchange/?term=type:app*&*
Allow: /en/matlabcentral/fileexchange/?term=type:class
Allow: /en/matlabcentral/fileexchange/?term=type:class&page=*
Disallow: /en/matlabcentral/fileexchange/?term=type:class*&*
Allow: /en/matlabcentral/fileexchange/?term=type:driver
Allow: /en/matlabcentral/fileexchange/?term=type:driver&page=*
Disallow: /en/matlabcentral/fileexchange/?term=type:driver*&*
Allow: /en/matlabcentral/fileexchange/?term=type:example
Allow: /en/matlabcentral/fileexchange/?term=type:example&page=*
Disallow: /en/matlabcentral/fileexchange/?term=type:example*&*
Allow: /en/matlabcentral/fileexchange/?term=type:function
Allow: /en/matlabcentral/fileexchange/?term=type:function&page=*
Disallow: /en/matlabcentral/fileexchange/?term=type:function*&*
Allow: /en/matlabcentral/fileexchange/?term=type:model
Allow: /en/matlabcentral/fileexchange/?term=type:model&page=*
Disallow: /en/matlabcentral/fileexchange/?term=type:model*&*
Allow: /en/matlabcentral/fileexchange/?term=type:script
Allow: /en/matlabcentral/fileexchange/?term=type:script&page=*
Disallow: /en/matlabcentral/fileexchange/?term=type:script*&*
Allow: /en/matlabcentral/fileexchange/?term=type:video
Allow: /en/matlabcentral/fileexchange/?term=type:video&page=*
Disallow: /en/matlabcentral/fileexchange/?term=type:video*&*
Disallow: /en/matlabcentral/fileexchange/authors/?
Allow: /en/matlabcentral/fileexchange/authors/?page=*
Disallow: /en/matlabcentral/fileexchange/authors/?page=*&*
Disallow: /en/matlabcentral/fileexchange/authors/contact/
Disallow: /en/matlabcentral/fileexchange/download.do
Disallow: /en/matlabcentral/fileexchange/feed.atom
Disallow: /en/matlabcentral/fileexchange/feedbacks
Disallow: /en/matlabcentral/fileexchange/index
Disallow: /en/matlabcentral/fileexchange/tags
Disallow: /en/matlabcentral/mlc-downloads/
Disallow: /en/matlabcentral/profile/api/v1/active/*
Disallow: /en/matlabcentral/profile/contact/*
Disallow: /en/moler/ncm-download.html
Disallow: /en/products/connections/portal/products/
Disallow: /en/products/connections/trials
Disallow: /en/quality/
Disallow: /en/searchresults*
Disallow: /en/searchresults/
Allow: /en/searchresults/*videos_all*videos_jp*facets
Disallow: /en/store/link/
Disallow: /en/support/bugreports/
Disallow: /en/support/pricing/
Disallow: /en/tagteam
Disallow: /en/test/
Disallow: /en/videos/*.feedback.html
Disallow: /en/videos/video-feedback.html
Disallow: /en/videos/video-feedback-confirmation.html
Disallow: /en/web_downloads


User-agent: Baiduspider
Disallow: /en/academia/books/search.html
Disallow: /en/company/user_stories/search.html
Disallow: /en/company/newsletters/search.html
Disallow: /en/hardware-support.html
Disallow: /en/hardware-support/instrument-control-toolbox/drivers-search.html
Disallow: /en/products/connections/search-products.html
Disallow: /en/products/connections/search-services.html
Disallow: /en/search.html
Disallow: /en/support/search.html
Disallow: /en/videos/search.html
Disallow: /en/searchresults*
Disallow: /en/searchresults/


User-agent: Slurp
Disallow: /en/company/aboutus/policies_statements/travel.html
Crawl-delay: 60

User-agent: nutch-solr-integration
Allow: /support/bugreports/ 

User-agent: Caliperbot/1.0 
Disallow: / 

User-agent: SearchmetricsBot 
Disallow: / 

User-agent: Covario-IDS/1.0 
Disallow: / 

User-agent: XoviBot 
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Googlebot
Disallow: /*.md$
User-agent: Bingbot
Disallow: /*.md$
User-agent: DuckDuckBot
Disallow: /*.md$
User-agent: Baiduspider
Disallow: /*.md$
User-agent: Sogou
Disallow: /*.md$
User-agent: NaverBot
Disallow: /*.md$