Skip to content

Commit

Permalink
feat: search return Uni, remove use of AuthCredentials
Browse files Browse the repository at this point in the history
  • Loading branch information
@astappiev
astappiev committed Sep 14, 2023
1 parent 8cea546 commit 4b3ec2b
Show file tree
Hide file tree
Showing 61 changed files with 710 additions and 890 deletions.
121 changes: 66 additions & 55 deletions connectors/BingConnector/src/main/java/de/l3s/interweb/connector/bing/BingConnector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,18 @@
@Dependent
public class BingConnector implements SearchConnector, SuggestConnector {
private static final Logger log = Logger.getLogger(BingConnector.class);
private static final int fallbackPerPageWeb = 50;
private static final int fallbackPerPageImages = 150;
private static final int fallbackPerPageVideos = 105;

@Inject
ObjectMapper mapper;

@RestClient
BingSearchClient searchClient;

@RestClient
BingSuggestClient suggestClient;

@Override
public String getName() {
Expand All @@ -39,69 +51,61 @@ public String getBaseUrl() {
return "https://bing.com/";
}

@Inject
ObjectMapper mapper;

@RestClient
BingSearchClient searchClient;

@RestClient
BingSuggestClient suggestClient;

@Override
public ContentType[] getSearchTypes() {
return new ContentType[]{ContentType.text, ContentType.image, ContentType.video};
return new ContentType[]{ContentType.webpages, ContentType.images, ContentType.videos, ContentType.news};
}

@Override
public SearchConnectorResults search(final SearchQuery query) throws ConnectorException {
BingResponse response = null;
public Uni<SearchConnectorResults> search(SearchQuery query) throws ConnectorException {
return processQuery(query).map(this::processResponse);
}

private Uni<BingResponse> processQuery(SearchQuery query) {
if (query.getContentTypes().size() == 1) {
if (query.getContentTypes().contains(ContentType.image)) {
response = searchClient.searchImages(
if (query.getContentTypes().contains(ContentType.images)) {
return searchClient.searchImages(
query.getQuery(),
query.getPerPage(150),
query.getOffset(150),
query.getPerPage(fallbackPerPageImages),
query.getOffset(fallbackPerPageImages),
query.getLanguage(),
BingUtils.getMarket(query.getLanguage()),
BingUtils.createFreshness(null, query.getDateTill())
).await().indefinitely();
} else if (query.getContentTypes().contains(ContentType.video)) {
response = searchClient.searchVideos(
BingUtils.createFreshness(null, query.getDateTo())
);
} else if (query.getContentTypes().contains(ContentType.videos)) {
return searchClient.searchVideos(
query.getQuery(),
query.getPerPage(105),
query.getOffset(105),
query.getPerPage(fallbackPerPageVideos),
query.getOffset(fallbackPerPageVideos),
query.getLanguage(),
BingUtils.getMarket(query.getLanguage()),
BingUtils.createFreshness(null, query.getDateTill())
).await().indefinitely();
BingUtils.createFreshness(null, query.getDateTo())
);
}
}

if (response == null) {
List<BingSearchClient.ResponseFilter> answerTypes = new ArrayList<>();
for (ContentType contentType : query.getContentTypes()) {
if (contentType == ContentType.text) {
answerTypes.add(BingSearchClient.ResponseFilter.webpages);
} else if (contentType == ContentType.image) {
answerTypes.add(BingSearchClient.ResponseFilter.images);
} else if (contentType == ContentType.video) {
answerTypes.add(BingSearchClient.ResponseFilter.videos);
}
List<BingSearchClient.ResponseFilter> answerTypes = new ArrayList<>();
for (ContentType contentType : query.getContentTypes()) {
if (contentType == ContentType.webpages) {
answerTypes.add(BingSearchClient.ResponseFilter.webpages);
} else if (contentType == ContentType.images) {
answerTypes.add(BingSearchClient.ResponseFilter.images);
} else if (contentType == ContentType.videos) {
answerTypes.add(BingSearchClient.ResponseFilter.videos);
} else if (contentType == ContentType.news) {
answerTypes.add(BingSearchClient.ResponseFilter.news);
}

response = searchClient.search(
query.getQuery(),
query.getPerPage(50),
query.getOffset(50),
query.getLanguage(),
BingUtils.getMarket(query.getLanguage()),
BingUtils.createFreshness(query.getDateFrom(), query.getDateTill()),
answerTypes.isEmpty() ? null : answerTypes.stream().map(Enum::name).collect(Collectors.joining(","))
).await().indefinitely();
}

return convertResponseToResults(response);
return searchClient.search(
query.getQuery(),
query.getPerPage(fallbackPerPageWeb),
query.getOffset(fallbackPerPageWeb),
query.getLanguage(),
BingUtils.getMarket(query.getLanguage()),
BingUtils.createFreshness(query.getDateFrom(), query.getDateTo()),
answerTypes.isEmpty() ? null : answerTypes.stream().map(Enum::name).collect(Collectors.joining(","))
);
}

@Override
Expand All @@ -119,7 +123,7 @@ public Uni<SuggestConnectorResults> suggest(SuggestQuery query) throws Connector
}));
}

private SearchConnectorResults convertResponseToResults(BingResponse response) throws ConnectorException {
private SearchConnectorResults processResponse(BingResponse response) throws ConnectorException {
if (response == null) {
throw new ConnectorException("No response");
}
Expand All @@ -128,20 +132,23 @@ private SearchConnectorResults convertResponseToResults(BingResponse response) t
throw new ConnectorException(response.getError().getMessage());
}

int rank = 0;
SearchConnectorResults results = new SearchConnectorResults();
if (response.getWebPages() != null && response.getWebPages().getValues() != null) {
WebPagesHolder webResults = response.getWebPages();
if (webResults.getCurrentOffset() != null) {
rank += webResults.getCurrentOffset();
}

if (webResults.getTotalEstimatedMatches() != null) {
results.addTotalResults(webResults.getTotalEstimatedMatches());
} else {
results.addTotalResults(webResults.getValues().size());
}

int index = 1;
for (WebPage page : webResults.getValues()) {
SearchItem resultItem = new SearchItem(index++);
resultItem.setType(ContentType.text);
SearchItem resultItem = new SearchItem(++rank);
resultItem.setType(ContentType.webpages);
resultItem.setTitle(page.getName());
resultItem.setDescription(page.getSnippet());
resultItem.setUrl(page.getUrl());
Expand All @@ -153,17 +160,19 @@ private SearchConnectorResults convertResponseToResults(BingResponse response) t

if (response.getImages() != null && response.getImages().getValues() != null) {
ImageHolder imagesResults = response.getImages();
if (imagesResults.getCurrentOffset() != null) {
rank += imagesResults.getCurrentOffset();
}

if (imagesResults.getTotalEstimatedMatches() != null) {
results.addTotalResults(imagesResults.getTotalEstimatedMatches());
} else {
results.addTotalResults(imagesResults.getValues().size());
}

int index = 1;
for (Image image : imagesResults.getValues()) {
SearchItem resultItem = new SearchItem(index++);
resultItem.setType(ContentType.image);
SearchItem resultItem = new SearchItem(++rank);
resultItem.setType(ContentType.images);
resultItem.setTitle(image.getName());
resultItem.setUrl(image.getContentUrl());
resultItem.setDate(DateUtils.parse(image.getDatePublished()));
Expand All @@ -188,17 +197,19 @@ private SearchConnectorResults convertResponseToResults(BingResponse response) t

if (response.getVideos() != null && response.getVideos().getValues() != null) {
VideoHolder videosResults = response.getVideos();
if (videosResults.getCurrentOffset() != null) {
rank += videosResults.getCurrentOffset();
}

if (videosResults.getTotalEstimatedMatches() != null) {
results.addTotalResults(videosResults.getTotalEstimatedMatches());
} else {
results.addTotalResults(videosResults.getValues().size());
}

int index = 1;
for (Video video : videosResults.getValues()) {
SearchItem resultItem = new SearchItem(index++);
resultItem.setType(ContentType.video);
SearchItem resultItem = new SearchItem(++rank);
resultItem.setType(ContentType.videos);
resultItem.setTitle(video.getName());
resultItem.setDescription(video.getDescription());
resultItem.setUrl(video.getContentUrl());
Expand Down
9 changes: 8 additions & 1 deletion connectors/BingConnector/src/main/java/de/l3s/interweb/connector/bing/BingSearchClient.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
package de.l3s.interweb.connector.bing;

import jakarta.validation.constraints.NotNull;
import jakarta.ws.rs.*;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;

Expand Down Expand Up @@ -120,6 +123,10 @@ Uni<BingResponse> searchVideos(

@ClientExceptionMapper
static RuntimeException toException(Response response) {
if (response.getStatus() == 429) {
return new ConnectorException("Rate exceeded");
}

return new ConnectorException("Remote service responded with HTTP " + response.getStatus(), response.readEntity(String.class));
}
}
22 changes: 11 additions & 11 deletions connectors/BingConnector/src/main/java/de/l3s/interweb/connector/bing/BingUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,26 +10,26 @@ public final class BingUtils {
* To get articles discovered by Bing during a specific timeframe, specify a date range in the form, YYYY-MM-DD..YYYY-MM-DD.
* To limit the results to a single date, set this parameter to a specific date, e.g. freshness=2019-02-04.
*/
static String createFreshness(LocalDate dateFrom, LocalDate dateTill) {
if (dateTill != null) {
if (dateTill.plusDays(2).isAfter(LocalDate.now())) { // add 1 day for safety
static String createFreshness(LocalDate from, LocalDate to) {
if (to != null) {
if (to.plusDays(2).isAfter(LocalDate.now())) { // add 1 day for safety
return "day";
} else if (dateTill.plusDays(9).isAfter(LocalDate.now())) { // add 2 days for safety
} else if (to.plusDays(9).isAfter(LocalDate.now())) { // add 2 days for safety
return "week";
} else if (dateTill.plusDays(34).isAfter(LocalDate.now())) { // add 3 days for safety
} else if (to.plusDays(34).isAfter(LocalDate.now())) { // add 3 days for safety
return "month";
}
}

if (dateFrom != null) {
String dateFromFormat = DateTimeFormatter.ISO_LOCAL_DATE.format(dateFrom);
if (from != null) {
String fromFormat = DateTimeFormatter.ISO_LOCAL_DATE.format(from);

if (dateTill != null && !dateFrom.equals(dateTill)) {
String dateTillFormat = DateTimeFormatter.ISO_LOCAL_DATE.format(dateTill);
return dateFromFormat + ".." + dateTillFormat;
if (to != null && !from.equals(to)) {
String toFormat = DateTimeFormatter.ISO_LOCAL_DATE.format(to);
return fromFormat + ".." + toFormat;
}

return dateFromFormat;
return fromFormat;
}

return null;
Expand Down
2 changes: 1 addition & 1 deletion ...gConnector/src/main/java/de/l3s/interweb/connector/bing/adapters/BingResponseAdapter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;

import de.l3s.interweb.connector.bing.entity.*;
import de.l3s.interweb.connector.bing.entity.Error;
import de.l3s.interweb.connector.bing.entity.*;

public class BingResponseAdapter extends StdDeserializer<BingResponse> {

Expand Down
9 changes: 0 additions & 9 deletions ...ctors/BingConnector/src/main/java/de/l3s/interweb/connector/bing/entity/BingResponse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ public class BingResponse {
private WebPagesHolder webPages;

private Error error;
private String jsonResponse;

public WebPagesHolder getWebPages() {
return webPages;
Expand Down Expand Up @@ -45,12 +44,4 @@ public Error getError() {
public void setError(final Error error) {
this.error = error;
}

public String getJsonResponse() {
return jsonResponse;
}

public void setJsonResponse(String jsonResponse) {
this.jsonResponse = jsonResponse;
}
}
29 changes: 18 additions & 11 deletions connectors/BingConnector/src/test/java/de/l3s/interweb/connector/bing/BingConnectorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,15 @@ void search() {
SearchQuery query = new SearchQuery();
query.setQuery("hello world");
query.setPerPage(20);
query.addContentType(ContentType.text);
query.addContentType(ContentType.webpages);
query.setDateFrom(LocalDate.of(2009, 1, 1));
query.setDateTill(LocalDate.of(2010, 6, 1));
query.setDateTo(LocalDate.of(2010, 6, 1));

SearchConnectorResults queryResult = connector.search(query);
SearchConnectorResults queryResult = connector.search(query).await().indefinitely();

int rank = 0;
for (SearchItem res : queryResult.getItems()) {
assertEquals(++rank, res.getRank());
log.info(res.toString());
}

Expand All @@ -65,14 +67,16 @@ void search() {
void searchImages() {
SearchQuery query = new SearchQuery();
query.setQuery("hannover");
query.addContentType(ContentType.image);
query.addContentType(ContentType.images);
query.setPerPage(30);
query.setPage(2);

SearchConnectorResults queryResult = connector.search(query);
SearchConnectorResults queryResult = connector.search(query).await().indefinitely();

int rank = 30;
for (SearchItem res : queryResult.getItems()) {
assertEquals(ContentType.image, res.getType());
assertEquals(++rank, res.getRank());
assertEquals(ContentType.images, res.getType());
log.info(res);
}

Expand All @@ -84,12 +88,15 @@ void searchImages() {
void searchVideos() {
SearchQuery query = new SearchQuery();
query.setQuery("hannover");
query.addContentType(ContentType.video);
query.addContentType(ContentType.videos);
query.setPerPage(30);

SearchConnectorResults queryResult = connector.search(query);
SearchConnectorResults queryResult = connector.search(query).await().indefinitely();

int rank = 0;
for (SearchItem res : queryResult.getItems()) {
assertEquals(ContentType.video, res.getType());
assertEquals(++rank, res.getRank());
assertEquals(ContentType.videos, res.getType());
log.info(res.toString());
}

Expand All @@ -100,8 +107,8 @@ void searchVideos() {
@Test
void bingSearchError() {
SearchQuery query = new SearchQuery();
query.setLanguage("error");
query.setPage(999999999);

assertThrowsExactly(ConnectorException.class, () -> connector.search(query));
assertThrowsExactly(ConnectorException.class, () -> connector.search(query).await().indefinitely());
}
}
Loading

0 comments on commit 4b3ec2b

Please sign in to comment.