improve issue main page

hakai_ckan_records_checks/__main__.py

@@ -4,11 +4,11 @@
 from pathlib import Path
 
 import click
-import numpy as np
 import pandas as pd
 from jinja2 import Environment, FileSystemLoader
 from loguru import logger
 from tqdm import tqdm
+import plotly.express as px
 
 from hakai_ckan_records_checks import hakai
 from hakai_ckan_records_checks.ckan import CKAN
@@ -126,10 +126,23 @@ def main(ckan_url, api_key, output, max_workers, log_level, cache):
     logger.info(f"Saving results to: '{output}'")
     results["catalog_summary"] = format_summary(results["catalog_summary"])
 
+    # Combine summary and issues
+    combined_issues = results['test_results'].merge(results['catalog_summary'], left_on='record_id', right_on='id').drop(columns=['id'])
+    standardized_issues = combined_issues.copy()
+    standardized_issues['message'] = standardized_issues['message'].str.replace('resources\[[0-9]+\]','resources[...]',regex=True)
+
+    # Generate figures
+    pie_chart = px.pie(standardized_issues, names='message',title='Hakai Records Issues Distribution')
+    pie_chart.update_traces(textposition='inside')
+    pie_chart.update_layout(uniformtext_minsize=12, uniformtext_mode='hide')
+    pie_chart_html = pie_chart.to_html(full_html=False)
+
     # save results
     Path(output).mkdir(parents=True, exist_ok=True)
     environment.get_template("index.html.jinja").stream(
         catalog_summary=format_summary(results["catalog_summary"]),
+        issues_pie_chart=pie_chart_html,
+        issues_table = combined_issues,
         time=pd.Timestamp.utcnow(),
         ckan_url=ckan_url,
     ).dump(f"{output}/index.html")
@@ -143,6 +156,9 @@ def main(ckan_url, api_key, output, max_workers, log_level, cache):
             time=pd.Timestamp.utcnow(),
         ).dump(f"{output}/issues/{record_id}.html")
 
+    # save results 
+    results['catalog_summary'].to_csv(f"{output}/catalog_summary.csv", index=False)
+    results['test_results'].to_csv(f"{output}/test_results.csv", index=False)
 
 if __name__ == "__main__":
     main()

hakai_ckan_records_checks/hakai.py

@@ -55,20 +55,20 @@ def _test(condition, level, message):
 
     # Review publisher
 
-    # Review ressources
-    for index, ressource in enumerate(record["resources"]):
-        _test(ressource["name"] != "", "ERROR", "Empty ressource name")
-        _test(ressource["url"] != "", "ERROR", "Empty ressource url")
-        _test(ressource["format"] != "", "ERROR", "Empty ressource format")
+    # Review resources
+    for index, resource in enumerate(record["resources"]):
+        _test(resource["name"] != "", "ERROR", "Empty resource name")
+        _test(resource["url"] != "", "ERROR", "Empty resource url")
+        _test(resource["format"] != "", "ERROR", "Empty resource format")
         _test(
-            ressource["format"] in ["HTML"],
+            resource["format"] in ["HTML","ERDDAP","OBIS"],
             "ERROR",
-            f"Invalid ressource format: resources[{index}].format={ressource['format']}",
+            f"Invalid resource format: resources[{index}].format={resource['format']}",
         )
         _test(
-            requests.get(ressource["url"]).status_code == 200,
+            requests.get(resource["url"]).status_code == 200,
             "ERROR",
-            f"Invalid ressources[{index}].url.status_code={requests.get(ressource['url']).status_code}",
+            f"Invalid resources[{index}].url.status_code={requests.get(resource['url']).status_code}",
         )
 
     # test spatial

hakai_ckan_records_checks/templates/index.html.jinja

@@ -8,6 +8,7 @@
      integrity="sha256-p4NxAoJBhIIN+hmNHrzRCf9tD/miZyoHS5obTRR9BMY="
      crossorigin=""/>
      <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
+     <script src="https://cdn.plot.ly/plotly-2.27.0.min.js" charset="utf-8"></script>
 </head>
 <style>
 #map { 
@@ -20,6 +21,8 @@ body {
   flex-direction: column;
   align-items: center;
   text-align: center;
+  max-width:1200px;
+  margin: auto;
 }
 h1 {
   margin: 30px;
@@ -40,22 +43,43 @@ h2 {
 }
 h3 {
   font-size: 20px;
-  background-color: lightgrey;
+  background-color: blue;
+  color: white;
   width: 100%;
   text-align: center;
   padding: 5px;
+  margin: 60px 0 30px 0
 }
 #table_wrapper {
     padding: 10px;
 }
+
+#pie-chart {
+    width: 100%;
+    heigh: 380px;
+}
+@media (min-width: 1200px) {
+  h3 {
+    border-radius: 10px;
+  }
+  #map {
+    border-radius: 10px;
+  }
+}
 </style>
 <body>
     <h1>Report Summary<br><a href="{{ckan_url}}">{{ckan_url}}</a></h1>
-    <p>This page present a summary of the different record hosted on the CKAN Catalogue.</p>
-    <div id='map'></div>
-    <h3>Records Summary</h3>
-    {{ catalog_summary.drop(columns=['id','name','spatial','vertical-extent','organization','private']).to_html( index=False, render_links=True, table_id='table', escape=False, classes='table table-striped table-hover table-sm')}}
+    <p>This page present a summary of: 
+    <br><a href="#records_summary">Records hosted on the catalogue</a> 
+    <br><a href="#issues_summary">Issues detected in some records</a>
+    </p>
 
+    <div id='map'></div>
+    <div id='pie-chart'>{{ issues_pie_chart }}</div>
+    <h3 id="records_summary">Records Summary</h3>
+    {{ catalog_summary.drop(columns=['id','name','spatial','vertical-extent','organization','private']).to_html( index=False, render_links=True, table_id='records_table', escape=False, classes='table table-striped table-hover table-sm')}}
+    <h3 id="issues_summary">Issues Summary</h3>
+    {{ issues_table[['title','level','message','links']].to_html( index=False, render_links=True, table_id='issues_table', escape=False, classes='table table-striped table-hover table-sm')}}
     <small>Last update: {{ time.isoformat() }}</small>
 </body>
 <!-- Make sure you put this AFTER Leaflet's CSS -->
@@ -88,7 +112,10 @@ h3 {
      <script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.10.25/js/jquery.dataTables.min.js"></script>
 <script>
 $(document).ready( function () {
-    $('#table').DataTable();
+    $('#records_table').DataTable();
+} );
+$(document).ready( function () {
+    $('#issues_table').DataTable();
 } );
 </script>
 </html>