Block web search indexing on backend; add tests (#574)

* Block web search indexing on backend; add tests

* Fix test URLs

ckan-backend-dev/src/ckanext-wri/ckanext/wri/templates/base.html

@@ -0,0 +1,6 @@
+{% ckan_extends %}
+
+{% block meta %}
+  {{ super() }}
+  <meta name="robots" content="noindex,nofollow">
+{% endblock %}

ckan-backend-dev/src/ckanext-wri/ckanext/wri/templates/home/robots.txt

@@ -0,0 +1,7 @@
+User-agent: *
+{% block all_user_agents -%}
+Disallow: /
+{%- endblock %}
+
+{% block additional_user_agents -%}
+{%- endblock %}

e2e-tests/cypress/e2e/robots_txt.cy.js

@@ -0,0 +1,12 @@
+describe('robots.txt Test', () => {
+  it('should only contain custom rule', () => {
+    cy.request('http://ckan-dev:5000/private-admin/robots.txt').then((response) => {
+      expect(response.status).to.eq(200);
+      expect(response.body).to.contain('User-agent: *');
+      expect(response.body).to.contain('Disallow: /');
+      expect(response.body).not.to.contain('Disallow: /dataset/rate/');
+      expect(response.body).not.to.contain('Disallow: /revision/');
+      expect(response.body).not.to.contain('Crawl-Delay: 10');
+    });
+  });
+});

e2e-tests/cypress/e2e/web_search_indexing.cy.js

@@ -0,0 +1,6 @@
+describe('Meta Robots Tag Test', () => {
+  it('should contain a "noindex, nofollow" robots meta tag on the homepage', () => {
+    cy.visit('http://ckan-dev:5000/private-admin/');
+    cy.get('head meta[name="robots"]').should('have.attr', 'content', 'noindex,nofollow');
+  });
+});