Introduction de matomo coté serveur

templates/misc/graph.part.html

@@ -1,17 +1,19 @@
 {% load i18n %}
-{% load dict_get %}
+{% load get_item %}
+{% load get_tuple_item %}
 
 
 <div id="{{ tab_name }}" class="tabcontent">
     <div class="stat-graph">
         <h3>{% trans graph_title %}</h3>
         <canvas
             id="{{ canvas_id }}"
-            {% for url in stats %}
-                data-views-{{ forloop.counter0 }}="[{% for view in url.stats|dict_get:metric %}{{view|dict_get:metric}} {% if not forloop.last %},{% endif %}{% endfor %}]"
-                data-label-views-{{ forloop.counter0 }}="{{url.label}}"
+                {% for url, report in reports.items %}
+                data-views-{{ forloop.counter0 }}="[{% for view in report|get_item:report_key|get_tuple_item:1 %}{{view}} {% if not forloop.last %},{% endif %}{% endfor %}]"
+                data-label-views-{{ forloop.counter0 }}="{{url.name}}"
+            data-time='[{% for view in report|get_item:report_key|get_tuple_item:0 %}"{{view}}"{% if not forloop.last %},{% endif %} {% endfor %}]'
             {% endfor %}
-            data-time='[{% for view in stats.0.stats|dict_get:metric %}"{{view.date}}"{% if not forloop.last %},{% endif %} {% endfor %}]'>
+            >
         </canvas>
     </div>
 </div>

templates/tutorialv2/stats/index.html

@@ -39,24 +39,19 @@ <h2 class="ico-after ico-tutorials">
                     {% endblocktrans %}
             {% endif %}
         </h2>
-
         <!-- Tab links -->
         <div class="tab">
             <span class="tablinks" tabindex="0" id="tab-view-graph">{% trans "Pages vues" %}</span>
-            <span class="tablinks" tabindex="0" id="tab-visit-time-graph">{% trans "Temps de lecture" %}</span>
-            <span class="tablinks" tabindex="0" id="tab-users-graph">{% trans "Nombre de visiteurs" %}</span>
-            <span class="tablinks" tabindex="0" id="tab-new-users-graph">{% trans "Nombre de nouveaux visiteurs" %}</span>
-            <span class="tablinks" tabindex="0" id="tab-sessions-graph">{% trans "Nombre de sessions" %}</span>
+            <span class="tablinks" tabindex="0" id="tab-visit-time-graph">{% trans "Temps moyen de lecture" %}</span>
+            <span class="tablinks" tabindex="0" id="tab-users-graph">{% trans "Nombre de visiteurs uniques" %}</span>
         </div>
         <!-- Tab content -->
-        {% include "misc/graph.part.html" with tab_name="tab-view-graph-content" graph_title="Évolution des pages vues sur le contenu" canvas_id="view-graph" metric="pageviews" %}
-        {% include "misc/graph.part.html" with tab_name="tab-visit-time-graph-content" graph_title="Évolution du temps de lecture (en secondes)" canvas_id="visit-time-graph" metric="avgTimeOnPage" %}
-        {% include "misc/graph.part.html" with tab_name="tab-users-graph-content" graph_title="Évolution du nombre de visiteurs" canvas_id="users-graph" metric="users" %}
-        {% include "misc/graph.part.html" with tab_name="tab-new-users-graph-content" graph_title="Évolution du nombre de nouveaux visiteurs" canvas_id="new-users-graph" metric="newUsers" %}
-        {% include "misc/graph.part.html" with tab_name="tab-sessions-graph-content" graph_title="Évolution du nombre de sessions" canvas_id="sessions-graph" metric="sessions" %}
+        {% include "misc/graph.part.html" with tab_name="tab-view-graph-content" graph_title="Évolution des pages vues sur le contenu" canvas_id="view-graph" report_key="nb_hits" %}
+        {% include "misc/graph.part.html" with tab_name="tab-visit-time-graph-content" graph_title="Évolution du temps moyen lecture (en secondes)" canvas_id="visit-time-graph" report_key="avg_time_on_page" %}
+        {% include "misc/graph.part.html" with tab_name="tab-users-graph-content" graph_title="Évolution du nombre de visiteurs uniques" canvas_id="users-graph" report_key="nb_uniq_visitors" %}
 
 
-        {% if cumulative_stats_by_url %}
+        {% if cumulative_stats %}
             {% if display == 'global' %}
                 {% if form.non_field_errors %}
                     <p class="content-wrapper alert-box warning">
@@ -71,32 +66,30 @@ <h2 class="ico-after ico-tutorials">
                     <th>{% trans "Partie" %}</th>
                     <th>{% trans "Vues" %}</th>
                     <th>{% trans "Temps moyen sur la page" %}</th>
-                    <th>{% trans "Visiteurs (Nouveaux)" %}</th>
-                    <th>{% trans "Sessions" %}</th>
+                    <th>{% trans "Visiteurs uniques" %}</th>
                     {% if display == 'global' %}<th></th>{% endif %}
                 </thead>
                 <tbody>
-                    {% for view in cumulative_stats_by_url %}
+                    {% for url, view in cumulative_stats.items %}
                         <tr>
-                            <td class="level-{{view.url.level}}">
-                                <a href="{{ view.url.url }}">{{ view.url.name }}</a>
+                            <td class="level-{{url.level}}">
+                                <a href="{{ url.url }}">{{ url.name }}</a>
                                 {% if display != 'details' %}
-                                - <a href="?urls={{ view.url.url }}{% if request.GET.start_date %}&start_date={{request.GET.start_date}}{% endif %}{% if request.GET.end_date %}&end_date={{request.GET.end_date}}{% endif %}" >{% trans "(détails)" %}</a>
+                                - <a href="?urls={{ url.url }}{% if request.GET.start_date %}&start_date={{request.GET.start_date}}{% endif %}{% if request.GET.end_date %}&end_date={{request.GET.end_date}}{% endif %}" >{% trans "(détails)" %}</a>
                                 {% endif %}
                             </td>
-                            <td>{{ view.pageviews }}</td>
-                            <td>{{ view.avgTimeOnPage|seconds_to_duration }}</td>
-                            <td>{{ view.users }} ({{ view.newUsers }})</td>
-                            <td>{{ view.sessions }}</td>
+                            <td>{{ view.nb_hits }}</td>
+                            <td>{{ view.avg_time_on_page|seconds_to_duration }}</td>
+                            <td>{{ view.nb_uniq_visitors }}</td>
                             {% if display == 'global' %}
-                                <td><input name="urls" id="id_urls_{{forloop.counter1}}" value="{{ view.url.url }}" type="checkbox"></td>
+                                <td><input name="urls" id="id_urls_{{forloop.counter1}}" value="{{ url.url }}" type="checkbox"></td>
                             {% endif %}
                         </tr>
                     {% endfor %}
                 </tbody>
             </table>
             {% if display == 'global' %}
-                <button type="submit" class="btn">{% trans "Comparer" %}</button>
+                <input type="submit" class="btn">{% trans "Comparer" %}</input>
                 </form>
             {% elif not content.has_extracts %}
                 <a href="{% url "content:stats-content" content.pk content.slug %}" class="btn btn-submit">{% trans "Revenir à la vue globale" %}</a>
@@ -107,11 +100,28 @@ <h2 class="ico-after ico-tutorials">
 
         <hr class="clearfix">
         <div>
-            <section class="content-col-2">
-                <h2>{% trans "Liens entrants vers ces pages" %}</h2>
+            <section class="content-col-3">
+                <h2>{% trans "Types de reférent vers ces pages" %}</h2>
+                <table class="stat-table fullwidth">
+                        <thead>
+                            <th>{% trans "Type" %}</th>
+                            <th>{% trans "Visites" %}</th>
+                        </thead>
+                        <tbody>
+                        {% for category, visits in type_referrers.items %}
+                            <tr>
+                                <td>{{ category }}</td>
+                                <td>{{ visits }}</td>
+                            </tr>
+                        {% endfor %}
+                        </tbody>
+                    </table>
+            </section>
+            <section class="content-col-3">
+                <h2>{% trans "Sites entrants vers ces pages" %}</h2>
                 <table class="stat-table fullwidth">
                     <thead>
-                        <th>{% trans "Liens" %}</th>
+                        <th>{% trans "Site" %}</th>
                         <th>{% trans "Visites" %}</th>
                     </thead>
                     <tbody>
@@ -124,8 +134,8 @@ <h2>{% trans "Liens entrants vers ces pages" %}</h2>
                     </tbody>
                 </table>
             </section>
-            <section class="content-col-2">
-                <h2>{% trans "Mots-clés des moteurs de recherche vers ces pages" %}</h2>
+            <section class="content-col-3">
+                <h2>{% trans "Mots-clés vers ces pages" %}</h2>
                 <table class="stat-table fullwidth">
                         <thead>
                             <th>{% trans "Mot-clés" %}</th>
@@ -147,11 +157,7 @@ <h2>{% trans "Mots-clés des moteurs de recherche vers ces pages" %}</h2>
             <h2>{% trans "Lexique" %}</h2>
             <h3>{% trans "Visiteurs" %}</h3>
             <p>
-                {% trans "Les Visiteurs, parfois appelé VU ou Visiteurs Uniques, correspondent au nombre d’utilisateurs uniques ayant consulté votre site. Un visiteur est considéré comme nouveau si c’est son premier passage sur la page. La mesure de Google est plutôt fiable sur le dédoublonnage grâce à ses outils de tracking poussés. Cependant elle ne doit pas être prise pour valeur exacte (l’outil n’étant par exemple pas considéré comme outil de mesure d’audience parfait par certains organismes officiels)." %}
-            </p>
-            <h3>{% trans "Sessions" %}</h3>
-            <p>
-                {% trans "Les sessions sont une mesure similaire au nombre de visites. Elles symbolisent le passage d’un visiteur. Tant que le visiteur est sur le site et actif, une seule session est comptée. Si l’utilisateur ne fait rien ou quitte le site pendant plus de 5 minutes, alors une nouvelle session démarre." %}
+                {% trans "Les Visiteurs, correspondent au nombre d’utilisateurs uniques ayant consulté votre site. La mesure de Matomo reste perfectible sur le dédoublonnage, autrement dit elle ne doit pas être prise pour valeur exacte (l’outil n’étant par exemple pas considéré comme outil de mesure d’audience parfait par certains organismes officiels)." %}
             </p>
             <h3>{% trans "Vues" %}</h3>
             <p>

zds/middlewares/matomomiddleware.py

@@ -0,0 +1,100 @@
+from _datetime import datetime
+from queue import Queue
+
+import requests
+import logging
+
+from django.conf import settings
+from threading import Thread
+
+from zds.member.views import get_client_ip
+
+matomo_token_auth = settings.ZDS_APP["site"]["matomoTokenAuth"]
+matomo_api_url = "{}/matomo.php?token_auth={}".format(settings.ZDS_APP["site"]["matomoUrl"], matomo_token_auth)
+matomo_site_id = settings.ZDS_APP["site"]["matomoSiteID"]
+matomo_api_version = 1
+logger = logging.getLogger(__name__)
+tracked_status_code = [200]
+tracked_methods = ["GET"]
+excluded_paths = ["/contenus", "/mp", "/munin", "/api"]
+
+
+def _background_process(queue: Queue):
+    data = queue.get(block=True)
+    while data:
+        params = {
+            "idsite": matomo_site_id,
+            "action_name": data["r_path"],
+            "rec": 1,
+            "apiv": matomo_api_version,
+            "lang": data["client_accept_language"],
+            "ua": data["client_user_agent"],
+            "urlref": data["client_referer"],
+            "url": data["client_url"],
+            "h": data["datetime"].hour,
+            "m": data["datetime"].minute,
+            "s": data["datetime"].second,
+        }
+        try:
+            if data["address_ip"] != "0.0.0.0":
+                params["cip"] = data["address_ip"]
+            requests.get(
+                matomo_api_url,
+                params=params,
+            )
+            logger.info(f'Matomo tracked this link : {data["client_url"]}')
+        except Exception:
+            logger.exception(f'Something went wrong with the tracking of the link {data["client_url"]}')
+
+        data = queue.get(block=True)
+
+
+class MatomoMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+        self.queue = Queue()
+        if settings.ZDS_APP["site"].get("matomo_enabled", True):
+            self.worker = Thread(target=_background_process, args=(self.queue,))
+            self.worker.setDaemon(True)
+            self.worker.start()
+
+    def __call__(self, request):
+        return self.process_response(request, self.get_response(request))
+
+    def matomo_track(self, request):
+        client_user_agent = request.META.get("HTTP_USER_AGENT", "")
+        client_referer = request.META.get("HTTP_REFERER", "")
+        client_accept_language = request.META.get("HTTP_ACCEPT_LANGUAGE", "")
+        client_url = f"{request.scheme}://{request.get_host()}{request.path}"
+        if settings.ZDS_APP["site"].get("matomo_enabled", True):
+            self.queue.put(
+                {
+                    "client_user_agent": client_user_agent,
+                    "client_referer": client_referer,
+                    "client_accept_language": client_accept_language,
+                    "client_url": client_url,
+                    "datetime": datetime.now().time(),
+                    "r_path": request.path,
+                    "address_ip": get_client_ip(request),
+                }
+            )
+
+    def process_response(self, request, response):
+        if response.status_code not in tracked_status_code or request.is_ajax():
+            return response
+        # only on get
+        if request.method in tracked_methods:
+            for p in excluded_paths:
+                if request.path.startswith(p):
+                    return response
+            try:
+                self.matomo_track(request)
+            except Exception:
+                logger.exception(f"Something failed with Matomo tracking system.")
+
+        return response
+
+    def __del__(self):
+        if settings.ZDS_APP["site"].get("matomo_enabled", True):
+            self.queue.put(False)
+            self.worker.join(timeout=2)

zds/settings/abstract_base/django.py

@@ -104,6 +104,7 @@
     # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
     "zds.utils.ThreadLocals",
     "zds.middlewares.setlastvisitmiddleware.SetLastVisitMiddleware",
+    "zds.middlewares.matomomiddleware.MatomoMiddleware",
     "zds.member.utils.ZDSCustomizeSocialAuthExceptionMiddleware",
 )
 

zds/settings/abstract_base/zds.py

@@ -68,6 +68,9 @@
         "la communauté.",
         "googleAnalyticsID": zds_config.get("google_analytics_id"),
         "googleTagManagerID": zds_config.get("google_tag_manager_id"),
+        "matomoSiteID": zds_config.get("matomo_site_id", 4),
+        "matomoUrl": zds_config.get("matomo_url", "https://matomo.zestedesavoir.com"),
+        "matomoTokenAuth": zds_config.get("matomo_token_auth", "2fa86dde2945cac5d1a7fa224e10e191"),
         "association": {
             "name": "Zeste de Savoir",
             "fee": zds_config.get("association_fee", "20 €"),

zds/settings/prod.py

@@ -97,19 +97,35 @@ def _get_version():
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": True,
+    "formatters": {
+        "verbose": {"format": "[%(levelname)s] -- %(asctime)s -- %(name)s : %(message)s"},
+    },
     "handlers": {
         "sentry": {
             "level": "WARNING",
             "class": "raven.handlers.logging.SentryHandler",
             "dsn": RAVEN_CONFIG["dsn"],
         },
+        "middlewares_log": {
+            "level": "INFO",
+            "class": "logging.handlers.RotatingFileHandler",
+            "maxBytes": 1024 * 1024 * 10,
+            "backupCount": 50,
+            "filename": "/var/log/zds/middlewares.log",
+            "formatter": "verbose",
+        },
     },
     "loggers": {
         "zds": {
             "handlers": ["sentry"],
             "propagate": True,
             "level": "DEBUG",
         },
+        "zds.middlewares": {
+            "handlers": ["middlewares_log"],
+            "propagate": True,
+            "level": "INFO",
+        },
         "zds.utils.templatetags.emarkdown": {
             "propagate": False,
             "level": "NOTSET",