spiders: move Statetul and OAI to common module

Signed-off-by: Szymon Łopaciuk <[email protected]>

hepcrawl/spiders/__init__.py

@@ -8,11 +8,3 @@
 # more details.
 
 from __future__ import absolute_import, division, print_function
-
-from scrapy import Spider
-
-
-class StatefulSpider(Spider):
-    def __init__(self, *args, **kwargs):
-        self.state = {}
-        super(StatefulSpider, self).__init__(*args, **kwargs)

hepcrawl/spiders/alpha_spider.py

@@ -18,7 +18,7 @@
 from scrapy import Request
 from scrapy.spiders import CrawlSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/aps_spider.py

@@ -18,7 +18,7 @@
 
 from scrapy import Request
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/arxiv_spider.py

@@ -16,7 +16,7 @@
 from scrapy import Request, Selector
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..mappings import CONFERENCE_WORDS, THESIS_WORDS

hepcrawl/spiders/base_spider.py

@@ -16,7 +16,7 @@
 from scrapy import Request
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/brown_spider.py

@@ -19,7 +19,7 @@
 from scrapy import Request
 from scrapy.spiders import CrawlSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/cds_spider.py

@@ -14,7 +14,7 @@
 from inspire_dojson import marcxml2record
 from os.path import join as path_join
 
-from .oaipmh_spider import OAIPMHSpider
+from .common import OAIPMHSpider
 from ..utils import ParsedItem
 
 

hepcrawl/spiders/common/__init__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of hepcrawl.
+# Copyright (C) 2015, 2016, 2017, 2018 CERN.
+#
+# hepcrawl is a free software; you can redistribute it and/or modify it
+# under the terms of the Revised BSD License; see LICENSE file for
+# more details.
+
+from __future__ import absolute_import, division, print_function
+
+from .oaipmh_spider import OAIPMHSpider
+from .stateful_spider import StatefulSpider

hepcrawl/spiders/oaipmh_spider.py → hepcrawl/spiders/common/oaipmh_spider.py

@@ -23,7 +23,7 @@
 
 from scrapy.http import Request, XmlResponse
 from scrapy.selector import Selector
-from . import StatefulSpider
+from .stateful_spider import StatefulSpider
 
 
 LOGGER = logging.getLogger(__name__)

hepcrawl/spiders/common/stateful_spider.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of hepcrawl.
+# Copyright (C) 2015, 2016, 2017, 2018 CERN.
+#
+# hepcrawl is a free software; you can redistribute it and/or modify it
+# under the terms of the Revised BSD License; see LICENSE file for
+# more details.
+
+from __future__ import absolute_import, division, print_function
+
+from scrapy import Spider
+
+
+class StatefulSpider(Spider):
+    def __init__(self, *args, **kwargs):
+        self.state = {}
+        super(StatefulSpider, self).__init__(*args, **kwargs)

hepcrawl/spiders/desy_spider.py

@@ -18,7 +18,7 @@
 from scrapy import Request
 from six.moves import urllib
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..utils import (
     ftp_list_files,
     ftp_connection_info,

hepcrawl/spiders/dnb_spider.py

@@ -14,7 +14,7 @@
 from scrapy import Request
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/edp_spider.py

@@ -19,7 +19,7 @@
 from scrapy import Request
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..extractors.jats import Jats
 from ..items import HEPRecord
 from ..loaders import HEPLoader

hepcrawl/spiders/elsevier_spider.py

@@ -23,7 +23,7 @@
 from scrapy import Request
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/hindawi_spider.py

@@ -14,7 +14,7 @@
 from scrapy import Request
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/infn_spider.py

@@ -19,7 +19,7 @@
 from scrapy.http import Request
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/iop_spider.py

@@ -18,7 +18,7 @@
 from scrapy import Request
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..extractors.nlm import NLM
 from ..items import HEPRecord
 from ..loaders import HEPLoader

hepcrawl/spiders/magic_spider.py

@@ -16,7 +16,7 @@
 from scrapy import Request
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/mit_spider.py

@@ -21,7 +21,7 @@
 from scrapy.http import Request
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/phenix_spider.py

@@ -16,7 +16,7 @@
 from scrapy import Request
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import ParsedItem

hepcrawl/spiders/phil_spider.py

@@ -17,7 +17,7 @@
 from scrapy import Request
 from scrapy.spiders import CrawlSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/pos_spider.py

@@ -18,7 +18,7 @@
 
 from scrapy import Request, Selector
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..dateutils import create_valid_date
 from ..items import HEPRecord
 from ..loaders import HEPLoader

hepcrawl/spiders/t2k_spider.py

@@ -16,7 +16,7 @@
 from scrapy import Request
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/wsp_spider.py

@@ -18,7 +18,7 @@
 from scrapy import Request
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..parsers import JatsParser
 from ..utils import (
     ftp_list_files,

tests/unit/test_oaipmh.py

@@ -11,7 +11,7 @@
 from mock import patch
 import pytest
 
-from hepcrawl.spiders.oaipmh_spider import OAIPMHSpider, NoLastRunToLoad
+from hepcrawl.spiders.common.oaipmh_spider import OAIPMHSpider, NoLastRunToLoad
 from hepcrawl.testlib.fixtures import clean_dir
 from scrapy.utils.project import get_project_settings