CDS: harvest directly from OAI-PMH

docker-compose.test.yml

@@ -18,6 +18,7 @@ services:
       - APP_CRAWLER_HOST_URL=http://scrapyd:6800
       - APP_API_PIPELINE_TASK_ENDPOINT_DEFAULT=hepcrawl.testlib.tasks.submit_results
       - APP_FILES_STORE=/tmp/file_urls
+      - APP_LAST_RUNS_PATH=/code/.scrapy/last_runs
       - APP_CRAWL_ONCE_PATH=/code/.scrapy
       - COVERAGE_PROCESS_START=/code/.coveragerc
       - BASE_USER_UID=${BASE_USER_UID:-1000}
@@ -58,8 +59,11 @@ services:
   functional_cds:
     <<: *service_base
     command: py.test -vv tests/functional/cds
-    links:
-      - scrapyd
+    depends_on:
+      scrapyd:
+        condition: service_healthy
+      cds-http-server.local:
+        condition: service_healthy
 
   functional_pos:
     <<: *service_base
@@ -129,6 +133,21 @@ services:
         - "CMD-SHELL"
         - "curl https://localhost:443/"
 
+  cds-http-server.local:
+    image: nginx:stable-alpine
+    volumes:
+      - ${PWD}/tests/functional/cds/fixtures/http_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf
+      - ${PWD}/tests/functional/cds/fixtures/http_server/records:/etc/nginx/html/
+    ports:
+      - 80:80
+    healthcheck:
+      timeout: 5s
+      interval: 5s
+      retries: 5
+      test:
+        - "CMD-SHELL"
+        - "curl http://localhost:80/"
+
   rabbitmq:
     image: rabbitmq
     healthcheck:

hepcrawl/downloaders.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of hepcrawl.
+# Copyright (C) 2016, 2017 CERN.
+#
+# hepcrawl is a free software; you can redistribute it and/or modify it
+# under the terms of the Revised BSD License; see LICENSE file for
+# more details.
+
+"""Additional downloaders."""
+
+
+from scrapy.http import Response
+
+
+class DummyDownloadHandler(object):
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def download_request(self, request, spider):
+        url = request.url
+        return Response(url, request=request)

hepcrawl/settings.py

@@ -19,6 +19,8 @@
 
 from __future__ import absolute_import, division, print_function
 
+from scrapy.settings import default_settings
+
 import os
 
 
@@ -40,6 +42,12 @@
     'http://localhost/schemas/records/'
 )
 
+# Location of last run information
+LAST_RUNS_PATH = os.environ.get(
+    'APP_LAST_RUNS_PATH',
+    '/var/lib/scrapy/last_runs/'
+)
+
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
 # CONCURRENT_REQUESTS=32
 
@@ -71,6 +79,13 @@
     'hepcrawl.middlewares.HepcrawlCrawlOnceMiddleware': 100,
 }
 
+# Configure custom downloaders
+# See https://doc.scrapy.org/en/0.20/topics/settings.html#download-handlers
+DOWNLOAD_HANDLERS = {
+    'oaipmh+http': 'hepcrawl.downloaders.DummyDownloadHandler',
+    'oaipmh+https': 'hepcrawl.downloaders.DummyDownloadHandler',
+}
+
 # Enable or disable downloader middlewares
 # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
 DOWNLOADER_MIDDLEWARES = {

hepcrawl/spiders/__init__.py

@@ -8,11 +8,3 @@
 # more details.
 
 from __future__ import absolute_import, division, print_function
-
-from scrapy import Spider
-
-
-class StatefulSpider(Spider):
-    def __init__(self, *args, **kwargs):
-        self.state = {}
-        return super(Spider, self).__init__(*args, **kwargs)

hepcrawl/spiders/alpha_spider.py

@@ -18,7 +18,7 @@
 from scrapy import Request
 from scrapy.spiders import CrawlSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/aps_spider.py

@@ -18,7 +18,7 @@
 
 from scrapy import Request
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/arxiv_spider.py

@@ -16,7 +16,7 @@
 from scrapy import Request, Selector
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..mappings import CONFERENCE_WORDS, THESIS_WORDS

hepcrawl/spiders/base_spider.py

@@ -16,7 +16,7 @@
 from scrapy import Request
 from scrapy.spiders import XMLFeedSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/brown_spider.py

@@ -19,7 +19,7 @@
 from scrapy import Request
 from scrapy.spiders import CrawlSpider
 
-from . import StatefulSpider
+from .common import StatefulSpider
 from ..items import HEPRecord
 from ..loaders import HEPLoader
 from ..utils import (

hepcrawl/spiders/cds_spider.py

@@ -9,67 +9,55 @@
 
 """Spider for the CERN Document Server OAI-PMH interface"""
 
-from scrapy.spider import XMLFeedSpider
-from scrapy import Request
-from harvestingkit.inspire_cds_package.from_cds import CDS2Inspire
-from harvestingkit.bibrecord import (
-    create_record as create_bibrec,
-    record_xml_output,
-)
-from dojson.contrib.marc21.utils import create_record
-from inspire_dojson.hep import hep
+import logging
+from flask.app import Flask
+from inspire_dojson import marcxml2record
+from os.path import join as path_join
 
-from . import StatefulSpider
+from .common import OAIPMHSpider
 from ..utils import ParsedItem
 
 
-class CDSSpider(StatefulSpider, XMLFeedSpider):
+LOGGER = logging.getLogger(__name__)
+
+
+class CDSSpider(OAIPMHSpider):
     """Spider for crawling the CERN Document Server OAI-PMH XML files.
 
     Example:
         Using OAI-PMH XML files::
 
-            $ scrapy crawl \\
-                cds \\
-                -a "source_file=file://$PWD/tests/functional/cds/fixtures/oai_harvested/cds_smoke_records.xml"
+            $ scrapy crawl CDS \\
+                -a "oai_set=forINSPIRE" -a "from_date=2017-10-10"
 
-    It uses `HarvestingKit <https://pypi.python.org/pypi/HarvestingKit>`_ to
-    translate from CDS's MARCXML into INSPIRE Legacy's MARCXML flavor. It then
-    employs `inspire-dojson <https://pypi.python.org/pypi/inspire-dojson>`_ to
-    transform the legacy INSPIRE MARCXML into the new INSPIRE Schema.
+    It uses `inspire-dojson <https://pypi.python.org/pypi/inspire-dojson>`_ to
+    translate from CDS's MARCXML into the new INSPIRE Schema.
     """
 
     name = 'CDS'
-    iterator = 'xml'
-    itertag = 'OAI-PMH:record'
-    namespaces = [
-        ('OAI-PMH', 'http://www.openarchives.org/OAI/2.0/'),
-        ('marc', 'http://www.loc.gov/MARC21/slim'),
-    ]
 
-    def __init__(self, source_file=None, **kwargs):
-        super(CDSSpider, self).__init__(**kwargs)
-        self.source_file = source_file
-
-    def start_requests(self):
-        yield Request(self.source_file)
+    def __init__(self,
+                 oai_endpoint='http://cds.cern.ch/oai2d',
+                 from_date=None,
+                 oai_set="forINSPIRE",
+                 *args, **kwargs):
+        super(CDSSpider, self).__init__(
+            url=oai_endpoint,
+            metadata_prefix='marcxml',
+            oai_set=oai_set,
+            from_date=from_date,
+            **kwargs
+        )
 
-    def parse_node(self, response, node):
-        node.remove_namespaces()
-        cds_bibrec, ok, errs = create_bibrec(
-            node.xpath('.//record').extract()[0]
+    def parse_record(self, selector):
+        selector.remove_namespaces()
+        record = selector.xpath('.//record').extract_first()
+        app = Flask('hepcrawl')
+        app.config.update(
+            self.settings.getdict('MARC_TO_HEP_SETTINGS', {})
         )
-        if not ok:
-            raise RuntimeError("Cannot parse record %s: %s", node, errs)
-        self.logger.info("Here's the record: %s" % cds_bibrec)
-        inspire_bibrec = CDS2Inspire(cds_bibrec).get_record()
-        marcxml_record = record_xml_output(inspire_bibrec)
-        record = create_record(marcxml_record)
-        json_record = hep.do(record)
-        base_uri = self.settings['SCHEMA_BASE_URI']
-        json_record['$schema'] = base_uri + 'hep.json'
-        parsed_item = ParsedItem(
-                record=json_record,
-                record_format='hep',
-            )
-        return parsed_item
+        with app.app_context():
+            json_record = marcxml2record(record)
+            base_uri = self.settings['SCHEMA_BASE_URI']
+            json_record['$schema'] = path_join(base_uri, 'hep.json')
+        return ParsedItem(record=json_record, record_format='hep')

hepcrawl/spiders/common/__init__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of hepcrawl.
+# Copyright (C) 2015, 2016, 2017, 2018 CERN.
+#
+# hepcrawl is a free software; you can redistribute it and/or modify it
+# under the terms of the Revised BSD License; see LICENSE file for
+# more details.
+
+from __future__ import absolute_import, division, print_function
+
+from .oaipmh_spider import OAIPMHSpider
+from .stateful_spider import StatefulSpider