use celerymonitor in CDS tests

Signed-off-by: Szymon Łopaciuk <[email protected]>
inspirehep · Dec 12, 2017 · 7b5a6ab · 7b5a6ab
1 parent 3c0a238
commit 7b5a6ab
Show file tree

Hide file tree

Showing 6 changed files with 1,606 additions and 72 deletions.
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -59,8 +59,11 @@ services:
   functional_cds:
     <<: *service_base
     command: py.test -vv tests/functional/cds
-    links:
-      - scrapyd
+    depends_on:
+      scrapyd:
+        condition: service_healthy
+      cds-http-server.local:
+        condition: service_healthy
 
   functional_pos:
     <<: *service_base
@@ -130,6 +133,21 @@ services:
         - "CMD-SHELL"
         - "curl https://localhost:443/"
 
+  cds-http-server.local:
+    image: nginx:stable-alpine
+    volumes:
+      - ${PWD}/tests/functional/cds/fixtures/http_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf
+      - ${PWD}/tests/functional/cds/fixtures/http_server/records:/etc/nginx/html/
+    ports:
+      - 80:80
+    healthcheck:
+      timeout: 5s
+      interval: 5s
+      retries: 5
+      test:
+        - "CMD-SHELL"
+        - "curl http://localhost:80/"
+
   rabbitmq:
     image: rabbitmq
     healthcheck:

diff --git a/hepcrawl/spiders/cds_spider.py b/hepcrawl/spiders/cds_spider.py
@@ -42,13 +42,18 @@ class CDSSpider(OAIPMHSpider):
 
     name = 'CDS'
 
-    def __init__(self, from_date=None, oai_set="forINSPIRE", *args, **kwargs):
+    def __init__(self,
+                 oai_endpoint='http://cds.cern.ch/oai2d',
+                 from_date=None,
+                 oai_set="forINSPIRE",
+                 *args, **kwargs):
         super(CDSSpider, self).__init__(
-            url='http://cds.cern.ch/oai2d',
+            url=oai_endpoint,
             metadata_prefix='marcxml',
             oai_set=oai_set,
             from_date=from_date,
-            **kwargs)
+            **kwargs
+        )
 
     def parse_record(self, selector):
         selector.remove_namespaces()

diff --git a/hepcrawl/spiders/oaipmh_spider.py b/hepcrawl/spiders/oaipmh_spider.py
@@ -51,8 +51,14 @@ class OAIPMHSpider(StatefulSpider):
     name = 'OAI-PMH'
     granularity = _Granularity.DATE
 
-    def __init__(self, url, metadata_prefix='marcxml', oai_set=None, alias=None,
-                 from_date=None, until_date=None, granularity=_Granularity.DATE,
+    def __init__(self,
+                 url,
+                 metadata_prefix='oai_dc',
+                 oai_set=None,
+                 alias=None,
+                 from_date=None,
+                 until_date=None,
+                 granularity=_Granularity.DATE,
                  record_class=Record, *args, **kwargs):
         super(OAIPMHSpider, self).__init__(*args, **kwargs)
         self.url = url

diff --git a/tests/functional/cds/fixtures/http_server/conf/proxy.conf b/tests/functional/cds/fixtures/http_server/conf/proxy.conf
@@ -0,0 +1,12 @@
+server {
+    listen                     80;
+    server_name                localhost;
+    charset_types              text/xml;
+    charset                    UTF-8;
+
+    location /oai2d {
+        if ($args ~ from=2017-11-15&verb=ListRecords&set=forINSPIRE&metadataPrefix=marcxml) {
+            rewrite ^.*$ /cds.xml permanent;
+        }
+    }
+}