feat: add tracking for search

l3s-learnweb · Nov 11, 2024 · 4a9880a · 4a9880a
1 parent c19da48
commit 4a9880a
Show file tree

Hide file tree

Showing 7 changed files with 40 additions and 12 deletions.
diff --git a/connectors/BingConnector/src/main/java/de/l3s/interweb/connector/bing/BingConnector.java b/connectors/BingConnector/src/main/java/de/l3s/interweb/connector/bing/BingConnector.java
@@ -144,6 +144,7 @@ private SearchConnectorResults processResponse(BingResponse response) throws Con
             processVideos(response.getVideos(), rank, results);
         }
 
+        results.setEstimatedCost(0.0234d); // $23.4015 EUR pro 1.000 Anrufe
         return results;
     }
 

diff --git a/interweb-core/src/main/java/de/l3s/interweb/core/search/SearchConnectorResults.java b/interweb-core/src/main/java/de/l3s/interweb/core/search/SearchConnectorResults.java
@@ -5,6 +5,7 @@
 
 import io.quarkus.runtime.annotations.RegisterForReflection;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 import de.l3s.interweb.core.ConnectorResults;
@@ -14,6 +15,8 @@ public class SearchConnectorResults extends ConnectorResults {
 
     @JsonProperty("total_results")
     private long totalResults = 0;
+    @JsonIgnore
+    private double estimatedCost = 0;
     private final List<SearchItem> items;
 
     public SearchConnectorResults() {
@@ -28,6 +31,14 @@ public void addResultItem(SearchItem resultItem) {
         items.add(resultItem);
     }
 
+    public double getEstimatedCost() {
+        return estimatedCost;
+    }
+
+    public void setEstimatedCost(double estimatedCost) {
+        this.estimatedCost = estimatedCost;
+    }
+
     public long getTotalResults() {
         return totalResults;
     }

diff --git a/interweb-server/src/main/java/de/l3s/interweb/server/features/api/ApiRequestSearch.java b/interweb-server/src/main/java/de/l3s/interweb/server/features/api/ApiRequestSearch.java
@@ -11,7 +11,6 @@
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 
-import de.l3s.interweb.core.search.ContentType;
 import de.l3s.interweb.server.features.user.User;
 
 @Entity
@@ -48,12 +47,12 @@ public class ApiRequestSearch extends PanacheEntityBase {
     @CreationTimestamp
     public Instant created;
 
-    public static ApiRequestSearch of(String engine, ContentType contentType, String query, Double estimatedCost, ApiKey apikey) {
+    public static ApiRequestSearch of(String engine, String contentType, String query, Double estimatedCost, ApiKey apikey) {
         ApiRequestSearch request = new ApiRequestSearch();
         request.user = apikey.user;
         request.apikey = apikey;
         request.engine = engine;
-        request.contentType = contentType.name();
+        request.contentType = contentType;
         request.query = query;
         request.estimatedCost = estimatedCost;
         return request;

diff --git a/interweb-server/src/main/java/de/l3s/interweb/server/features/api/UsageService.java b/interweb-server/src/main/java/de/l3s/interweb/server/features/api/UsageService.java
@@ -58,13 +58,13 @@ public Uni<Double> deduct(User user, double cost) {
     @WithSession
     @ConsumeEvent("api-request-chat")
     public Uni<Void> consumeChatRequest(ApiRequestChat request) {
-        return request.persist().eventually(() -> deduct(request.user, request.estimatedCost)).replaceWithVoid();
+        return request.persistAndFlush().eventually(() -> deduct(request.user, request.estimatedCost)).replaceWithVoid();
     }
 
     @WithSession
     @ConsumeEvent("api-request-search")
     public Uni<Void> consumeSearchRequest(ApiRequestSearch request) {
-        return request.persist().eventually(() -> deduct(request.user, request.estimatedCost)).replaceWithVoid();
+        return request.persistAndFlush().eventually(() -> deduct(request.user, request.estimatedCost)).replaceWithVoid();
     }
 
     private static class UsageValue {

diff --git a/interweb-server/src/main/java/de/l3s/interweb/server/features/search/SearchResource.java b/interweb-server/src/main/java/de/l3s/interweb/server/features/search/SearchResource.java
@@ -28,6 +28,7 @@
 import de.l3s.interweb.core.search.*;
 import de.l3s.interweb.core.util.StringUtils;
 import de.l3s.interweb.server.Roles;
+import de.l3s.interweb.server.features.api.ApiKey;
 
 @Tag(name = "Search", description = "Search internet by query")
 @Path("/search")
@@ -72,13 +73,15 @@ public Uni<SearchResults> search(@Parameter(description = "The search query", ex
     @POST
     @PermissionsAllowed("search")
     public Uni<SearchResults> search(@NotNull @Valid SearchQuery query, @HeaderParam("Cache-Control") String cacheControl) {
+        ApiKey apikey = securityIdentity.getCredential(ApiKey.class);
+
         long start = System.currentTimeMillis();
         if (NO_CACHE.equals(cacheControl)) {
             query.setIgnoreCache(true);
         }
 
         query.setId(UUID.randomUUID().toString());
-        return searchService.search(query).map(results -> {
+        return searchService.search(query, apikey).map(results -> {
             results.setQuery(query);
             results.setElapsedTime(System.currentTimeMillis() - start);
             return results;

diff --git a/interweb-server/src/main/java/de/l3s/interweb/server/features/search/SearchService.java b/interweb-server/src/main/java/de/l3s/interweb/server/features/search/SearchService.java
@@ -14,19 +14,29 @@
 import io.quarkus.cache.CompositeCacheKey;
 import io.smallrye.mutiny.Multi;
 import io.smallrye.mutiny.Uni;
+import io.vertx.core.eventbus.EventBus;
 import org.jboss.logging.Logger;
 
 import de.l3s.interweb.core.ConnectorException;
 import de.l3s.interweb.core.search.SearchConnector;
 import de.l3s.interweb.core.search.SearchConnectorResults;
 import de.l3s.interweb.core.search.SearchQuery;
 import de.l3s.interweb.core.search.SearchResults;
+import de.l3s.interweb.server.features.api.ApiKey;
+import de.l3s.interweb.server.features.api.ApiRequestSearch;
+import de.l3s.interweb.server.features.api.UsageService;
 
 @ApplicationScoped
 public class SearchService {
     private static final Logger log = Logger.getLogger(SearchService.class);
     private static final int defaultTimeout = 10_000;
 
+    @Inject
+    EventBus bus;
+
+    @Inject
+    UsageService usageService;
+
     @Inject
     @CacheName("search")
     Cache cache;
@@ -58,24 +68,28 @@ private Collection<SearchConnector> getConnectors(Set<String> services) {
         return this.providers.values();
     }
 
-    public Uni<SearchResults> search(SearchQuery query) {
+    public Uni<SearchResults> search(SearchQuery query, ApiKey apikey) {
         Duration timeout = Duration.ofMillis(Objects.requireNonNullElse(query.getTimeout(), defaultTimeout));
         return Multi.createFrom()
             .iterable(getConnectors(query.getServices()))
-            .onItem().transformToUniAndMerge(connector -> searchIn(query, connector, timeout))
+            .onItem().transformToUniAndMerge(connector -> searchIn(query, connector, timeout, apikey))
             .collect().asList().map(SearchResults::new);
     }
 
-    private Uni<SearchConnectorResults> searchIn(SearchQuery query, SearchConnector connector, Duration timeout) {
+    private Uni<SearchConnectorResults> searchIn(SearchQuery query, SearchConnector connector, Duration timeout, ApiKey apikey) {
         long start = System.currentTimeMillis();
-        return searchWithCache(query, connector)
+        return usageService.allocate(apikey.user)
+            .chain(() -> searchWithCache(query, connector))
             .ifNoItem().after(timeout).failWith(new ConnectorException(connector.getName() + " reached timeout after " + timeout.toMillis() + "ms"))
             .onFailure(ConnectorException.class).recoverWithItem(failure -> {
                 log.error("Error in search connector " + connector.getId(), failure);
                 SearchConnectorResults results = new SearchConnectorResults();
                 results.setError((ConnectorException) failure);
                 return results;
-            }).onItem().invoke(conRes -> connector.fillResult(conRes, System.currentTimeMillis() - start));
+            }).onItem().invoke(conRes -> {
+                connector.fillResult(conRes, System.currentTimeMillis() - start);
+                bus.send("api-request-search", ApiRequestSearch.of(connector.getName(), query.getContentTypes().toString(), query.getQuery(), conRes.getEstimatedCost(), apikey));
+            });
     }
 
     private Uni<SearchConnectorResults> searchWithCache(SearchQuery query, SearchConnector connector) {

diff --git a/interweb-server/src/test/java/de/l3s/interweb/server/RequestApiKeyAuthTest.java b/interweb-server/src/test/java/de/l3s/interweb/server/RequestApiKeyAuthTest.java
@@ -42,7 +42,7 @@ public void setup() {
         SearchQuery searchQuery = new SearchQuery();
         searchQuery.setQuery("hello world");
         searchQuery.setContentTypes(de.l3s.interweb.core.search.ContentType.image);
-        Mockito.when(searchService.search(searchQuery)).thenReturn(Uni.createFrom().item(new SearchResults()));
+        Mockito.when(searchService.search(searchQuery, testKey)).thenReturn(Uni.createFrom().item(new SearchResults()));
     }
 
     @Test
-Original file line number
+Diff line change
@@ Expand Up @@
                 processVideos(response.getVideos(), rank, results);
             }
+            results.setEstimatedCost(0.0234d); // $23.4015 EUR pro 1.000 Anrufe
             return results;
         }
@@ Expand Down @@