disable Pagination with rel="next" and rel="prev" python
from scrapy.conf import settings def disable_pagination_middleware(request, spider): if 'no_pagination' in request.meta: request.meta['dont_redirect'] = True request.meta['handle_httpstatus_list'] = [301, 302] class DisablePaginationMiddleware(object): def process_request(self, request, spider): disable_pagination_middleware(request, spider)