# In the new version (scrapy 1.1) launched 2016-05-11 the crawl first downloads robots.txt
# before crawling. To change this behavior change in your settings.py with ROBOTSTXT_OBEY
ROBOTSTXT_OBEY = False# In the new version (scrapy 1.1) launched 2016-05-11 the crawl first downloads robots.txt
# before crawling. To change this behavior change in your settings.py with ROBOTSTXT_OBEY
ROBOTSTXT_OBEY = False# In the new version (scrapy 1.1) launched 2016-05-11 the crawl first downloads robots.txt
# before crawling. To change this behavior change in your settings.py with ROBOTSTXT_OBEY
ROBOTSTXT_OBEY = False# In the new version (scrapy 1.1) launched 2016-05-11 the crawl first downloads robots.txt
# before crawling. To change this behavior change in your settings.py with ROBOTSTXT_OBEY
ROBOTSTXT_OBEY = False# In the new version (scrapy 1.1) launched 2016-05-11 the crawl first downloads robots.txt
# before crawling. To change this behavior change in your settings.py with ROBOTSTXT_OBEY
ROBOTSTXT_OBEY = False# In the new version (scrapy 1.1) launched 2016-05-11 the crawl first downloads robots.txt
# before crawling. To change this behavior change in your settings.py with ROBOTSTXT_OBEY
ROBOTSTXT_OBEY = False# In the new version (scrapy 1.1) launched 2016-05-11 the crawl first downloads robots.txt
# before crawling. To change this behavior change in your settings.py with ROBOTSTXT_OBEY
ROBOTSTXT_OBEY = False# In the new version (scrapy 1.1) launched 2016-05-11 the crawl first downloads robots.txt
# before crawling. To change this behavior change in your settings.py with ROBOTSTXT_OBEY
ROBOTSTXT_OBEY = False# In the new version (scrapy 1.1) launched 2016-05-11 the crawl first downloads robots.txt
# before crawling. To change this behavior change in your settings.py with ROBOTSTXT_OBEY
ROBOTSTXT_OBEY = False# In the new version (scrapy 1.1) launched 2016-05-11 the crawl first downloads robots.txt
# before crawling. To change this behavior change in your settings.py with ROBOTSTXT_OBEY
ROBOTSTXT_OBEY = False