Skip to content

Commit

Permalink
fix: proper models creation
Browse files Browse the repository at this point in the history
  • Loading branch information
astappiev committed Jul 29, 2024
1 parent 6e7e662 commit c9a5e09
Show file tree
Hide file tree
Showing 6 changed files with 38 additions and 48 deletions.
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package de.l3s.interweb.connector.anthropic;

import java.time.Instant;
import java.time.LocalDate;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
Expand Down Expand Up @@ -28,13 +29,13 @@ public class AnthropicConnector implements ChatConnector {
* https://docs.anthropic.com/en/docs/about-claude/models
*/
private static final List<Model> models = List.of(
Model.of("claude-3-5-sonnet-20240620", "anthropic", new UsagePrice(0.003, 0.015), Instant.parse("2024-06-20")),
Model.of("claude-3-opus-20240229", "anthropic", new UsagePrice(0.015, 0.075), Instant.parse("2024-02-29")),
Model.of("claude-3-sonnet-20240229", "anthropic", new UsagePrice(0.003, 0.015), Instant.parse("2024-02-29")),
Model.of("claude-3-haiku-20240307", "anthropic", new UsagePrice(0.00025, 0.00125), Instant.parse("2024-03-07")),
Model.of("claude-2.1", "anthropic", new UsagePrice(0.008, 0.024), Instant.parse("2023-11-23")),
Model.of("claude-2.0", "anthropic", new UsagePrice(0.008, 0.024), Instant.parse("2023-07-11")),
Model.of("claude-instant-1.2", "anthropic", new UsagePrice(0.0008, 0.0024), Instant.parse("2023-08-09"))
Model.of("claude-3-5-sonnet-20240620", "anthropic", new UsagePrice(0.003, 0.015), LocalDate.of(2024, 6, 20)),
Model.of("claude-3-opus-20240229", "anthropic", new UsagePrice(0.015, 0.075), LocalDate.of(2024, 2, 29)),
Model.of("claude-3-sonnet-20240229", "anthropic", new UsagePrice(0.003, 0.015), LocalDate.of(2024, 2, 29)),
Model.of("claude-3-haiku-20240307", "anthropic", new UsagePrice(0.00025, 0.00125), LocalDate.of(2024, 3, 7)),
Model.of("claude-2.1", "anthropic", new UsagePrice(0.008, 0.024), LocalDate.of(2023, 11, 23)),
Model.of("claude-2.0", "anthropic", new UsagePrice(0.008, 0.024), LocalDate.of(2023, 7, 11)),
Model.of("claude-instant-1.2", "anthropic", new UsagePrice(0.0008, 0.0024), LocalDate.of(2023, 8, 9))
);

@RestClient
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package de.l3s.interweb.connector.openai;

import java.time.Instant;
import java.time.LocalDate;
import java.util.List;
import java.util.Optional;

Expand Down Expand Up @@ -28,12 +28,12 @@ public class OpenaiConnector implements ChatConnector {
* https://azure.microsoft.com/de-de/pricing/details/cognitive-services/openai-service/
*/
private static final List<Model> models = List.of(
Model.of("gpt-35-turbo", "openai", new UsagePrice(0.002, 0.002), Instant.parse("2023-03-01")),
Model.of("gpt-35-turbo-16k", "openai", new UsagePrice(0.003, 0.004), Instant.parse("2023-06-13")),
Model.of("gpt-35-turbo-1106", "openai", new UsagePrice(0.001, 0.002), Instant.parse("2023-11-06")),
Model.of("gpt-4", "openai", new UsagePrice(0.03, 0.06), Instant.parse("2023-06-13")),
Model.of("gpt-4-turbo", "openai", new UsagePrice(0.01, 0.03), Instant.parse("2024-01-25")),
Model.of("gpt-4o", "openai", new UsagePrice(0.005, 0.015), Instant.parse("2024-05-13"))
Model.of("gpt-35-turbo", "openai", new UsagePrice(0.002, 0.002), LocalDate.of(2023, 3, 1)),
Model.of("gpt-35-turbo-16k", "openai", new UsagePrice(0.003, 0.004), LocalDate.of(2023, 6, 13)),
Model.of("gpt-35-turbo-1106", "openai", new UsagePrice(0.001, 0.002), LocalDate.of(2023, 11, 6)),
Model.of("gpt-4", "openai", new UsagePrice(0.03, 0.06), LocalDate.of(2023, 6, 13)),
Model.of("gpt-4-turbo", "openai", new UsagePrice(0.01, 0.03), LocalDate.of(2024, 1, 25)),
Model.of("gpt-4o", "openai", new UsagePrice(0.005, 0.015), LocalDate.of(2024, 5, 13))
);

@RestClient
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package de.l3s.interweb.core.models;

import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;

import io.quarkus.runtime.annotations.RegisterForReflection;

Expand Down Expand Up @@ -70,12 +72,14 @@ public void setCreated(Instant created) {
this.created = created;
}

public static Model of(String id, String ownedBy, UsagePrice price, Instant created) {
public static Model of(String id, String ownedBy, UsagePrice price, LocalDate created) {
Model model = new Model();
model.setId(id);
model.setOwnedBy(ownedBy);
model.setPrice(price);
model.setCreated(created);
if (created != null) {
model.setCreated(created.atStartOfDay(ZoneId.systemDefault()).toInstant());
}
return model;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@

import io.quarkus.runtime.annotations.RegisterForReflection;

import com.fasterxml.jackson.annotation.JsonInclude;

@RegisterForReflection
@JsonInclude(JsonInclude.Include.NON_NULL)
public class UsagePrice {
private double input;
private double output;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
package de.l3s.interweb.server.features.models;

import java.util.*;
import java.util.concurrent.CompletableFuture;

import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;

import io.quarkus.arc.All;
import io.quarkus.cache.Cache;
import io.quarkus.cache.CacheName;
import io.quarkus.cache.CacheResult;
import io.quarkus.cache.CaffeineCache;
import io.smallrye.mutiny.Multi;
import io.smallrye.mutiny.Uni;
import org.jboss.logging.Logger;
Expand All @@ -22,10 +18,6 @@
public class ModelsService {
private static final Logger log = Logger.getLogger(ModelsService.class);

@Inject
@CacheName("model")
Cache cache;

private final Map<String, ModelsConnector> providers;

@Inject
Expand Down Expand Up @@ -54,37 +46,25 @@ public Collection<ModelsConnector> getConnectors() {
/**
* Get all models from all providers, could be time expensive as each connector calls to their own source of data and waits for the response.
*/
@CacheResult(cacheName = "models") // this is short cache, to avoid spamming the providers
@CacheResult(cacheName = "models") // this is a short-term cache, to avoid spamming the providers
public Uni<List<Model>> getModels() {
CaffeineCache cache = this.cache.as(CaffeineCache.class);

return Multi.createFrom().iterable(providers.values())
// invalidate cache
.call(cache::invalidateAll)
.onItem().transformToUniAndMerge(connector -> {
return connector.getModels()
// populate cache with models
.invoke(models -> models.forEach(model -> {
model.setId(model.getId().toLowerCase(Locale.ROOT));
model.setProvider(connector.getId());
cache.put(model.getId(), CompletableFuture.completedFuture(model));
}));
})
.collect().in(ArrayList::new, List::addAll);
.onItem().transformToUniAndMerge(connector -> connector.getModels()
.invoke(models -> models.forEach(model -> {
model.setId(model.getId().toLowerCase(Locale.ROOT));
model.setProvider(connector.getId());
})))
.collect()
.in(ArrayList::new, List::addAll);
}

/**
* The method implements a cache for the models. If the model is not in the cache, it will be fetched from the providers.
*/
@CacheResult(cacheName = "model")
public Uni<Model> getModel(String modelId) {
CaffeineCache cache = this.cache.as(CaffeineCache.class);
modelId = modelId.toLowerCase(Locale.ROOT);

CompletableFuture<Model> value = cache.getIfPresent(modelId);
if (value != null) {
return Uni.createFrom().future(value);
}

return getModels().replaceWith(Uni.createFrom().future(cache.getIfPresent(modelId)));
return getModels().map(models -> models.stream()
.filter(model -> model.getId().equalsIgnoreCase(modelId.toLowerCase(Locale.ROOT)))
.findFirst().orElse(null));
}
}
2 changes: 2 additions & 0 deletions interweb-server/src/main/resources/application.properties
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ quarkus.http.cors.access-control-allow-credentials=true
quarkus.ssl.native=true
quarkus.http.port=8080

quarkus.log.level=INFO

quarkus.cache.caffeine.initial-capacity=100
quarkus.cache.caffeine.maximum-size=1000
quarkus.cache.caffeine.expire-after-write=P7D
Expand Down

0 comments on commit c9a5e09

Please sign in to comment.