Skip to content
This repository has been archived by the owner on Jan 3, 2019. It is now read-only.

Connection problem (partial) fix #80

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Connection problem (partial) fix #80

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions fcrepo-message-consumer-core/src/main/java/org/fcrepo/indexer/IndexerGroup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
//import org.apache.http.impl.NoConnectionReuseStrategy;
import org.fcrepo.kernel.utils.EventType;
import org.slf4j.Logger;

Expand All @@ -54,7 +55,7 @@
import static com.hp.hpl.jena.rdf.model.ResourceFactory.createProperty;
import static com.hp.hpl.jena.rdf.model.ResourceFactory.createResource;
import static com.hp.hpl.jena.vocabulary.RDF.type;
import static java.lang.Integer.MAX_VALUE;
//import static java.lang.Integer.MAX_VALUE;
import static javax.jcr.observation.Event.NODE_REMOVED;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.fcrepo.kernel.FedoraJcrTypes.FCR_METADATA;
Expand Down Expand Up @@ -206,12 +207,13 @@ protected DefaultHttpClient httpClient(final String repositoryURL) {
}

final PoolingClientConnectionManager connMann = new PoolingClientConnectionManager();
connMann.setMaxTotal(MAX_VALUE);
connMann.setDefaultMaxPerRoute(MAX_VALUE);
connMann.setMaxTotal(200);
connMann.setDefaultMaxPerRoute(20);

final DefaultHttpClient httpClient = new DefaultHttpClient(connMann);
httpClient.setRedirectStrategy(new DefaultRedirectStrategy());
httpClient.setHttpRequestRetryHandler(new StandardHttpRequestRetryHandler(0, false));
//httpClient.setReuseStrategy(new NoConnectionReuseStrategy());

// If the Fedora instance requires authentication, set it up here
if (!isBlank(fedoraUsername) && !isBlank(fedoraPassword)) {
Expand Down
27 changes: 17 additions & 10 deletions fcrepo-message-consumer-core/src/main/java/org/fcrepo/indexer/JcrXmlRetriever.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;

Expand All @@ -39,6 +38,7 @@

/**
* Retrieves Modeshape jcr/xml for file system persistence
*
* @author lsitu
*/
public class JcrXmlRetriever implements Supplier<InputStream> {
Expand All @@ -51,8 +51,9 @@ public class JcrXmlRetriever implements Supplier<InputStream> {

/**
* Constructor
*
* @param identifier the URI identifier
* @param client the http client
* @param client the http client
*/
public JcrXmlRetriever(final URI identifier, final HttpClient client) {
this.identifier = identifier;
Expand All @@ -64,37 +65,43 @@ public JcrXmlRetriever(final URI identifier, final HttpClient client) {
* Retrieve jcr/xml with no binary contents from the repository
*/
public InputStream get() {
final HttpHead headRequest = new HttpHead(identifier);
HttpGet request = null;

try {
// make an initial HEAD request and check Link headers for descriptions located elsewhere
final HttpHead headRequest = new HttpHead(identifier);
final HttpResponse headResponse = httpClient.execute(headRequest);
URI descriptionURI = null;
final Header[] links = headResponse.getHeaders("Link");
if ( links != null ) {
for ( Header h : headResponse.getHeaders("Link") ) {
if (links != null) {
for (Header h : headResponse.getHeaders("Link")) {
final Link link = Link.valueOf(h.getValue());
if ( link.getRel().equals("describedby") ) {
if (link.getRel().equals("describedby")) {
descriptionURI = link.getUri();
LOGGER.debug("Using URI from Link header: {}", descriptionURI);
}
}
}
if ( descriptionURI == null ) {
if (descriptionURI == null) {
descriptionURI = identifier;
}

final HttpUriRequest request = new HttpGet(descriptionURI.toString() + "/fcr:export?skipBinary=true");
request = new HttpGet(descriptionURI.toString() + "/fcr:export?skipBinary=true");
LOGGER.debug("Retrieving jcr/xml content from: {}...", request.getURI());
final HttpResponse response = httpClient.execute(request);
if (response.getStatusLine().getStatusCode() == SC_OK) {
return response.getEntity().getContent();
return response.getEntity().getContent();
} else {
throw new HttpException(response.getStatusLine().getStatusCode() + " : " +
EntityUtils.toString(response.getEntity()));
EntityUtils.toString(response.getEntity()));
}
} catch (IOException | HttpException e) {
throw propagate(e);
} finally {
headRequest.releaseConnection();
if (request != null) {
request.releaseConnection();
}
}
}

Expand Down
60 changes: 36 additions & 24 deletions fcrepo-message-consumer-core/src/main/java/org/fcrepo/indexer/NamedFieldsRetriever.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,46 +62,48 @@ public class NamedFieldsRetriever implements Supplier<NamedFields> {

private Gson gson;

private static final Type typeToken = new TypeToken<NamedFields>() {}
private static final Type typeToken = new TypeToken<NamedFields>() {
}
.getType();

private static final Logger LOGGER = getLogger(NamedFieldsRetriever.class);

/**
* @param uri the URI identifier
* @param uri the URI identifier
* @param client the http client
* @param rdfr Used to determine the transform to use with this indexing
* step
* @param rdfr Used to determine the transform to use with this indexing
* step
*/
public NamedFieldsRetriever(final URI uri, final HttpClient client,
final Supplier<Model> rdfr) {
final Supplier<Model> rdfr) {
this.uri = uri;
this.httpClient = client;
this.rdfr = rdfr;
final NamedFieldsDeserializer deserializer =
new NamedFieldsDeserializer();
new NamedFieldsDeserializer();
this.gson =
new GsonBuilder().registerTypeAdapter(typeToken, deserializer)
.create();
new GsonBuilder().registerTypeAdapter(typeToken, deserializer)
.create();
deserializer.setGson(gson);
}

@Override
public NamedFields get() {
LOGGER.debug("Retrieving RDF representation for: {}", uri);
final HttpHead headRequest = new HttpHead(uri);

try {
final Model rdf = rdfr.get();

// If there is no transform-predicate on this resource, look deeper...
if (!rdf.contains(createResource(uri.toString()), INDEXING_TRANSFORM_PREDICATE)) {
LOGGER.info("Looking up property locating LDPath transform for: {}", uri);
// make an initial HEAD request and check Link headers for descriptions located elsewhere
final HttpHead headRequest = new HttpHead(uri);
final HttpResponse headResponse = httpClient.execute(headRequest);
URI descriptionURI = null;
final Header[] links = headResponse.getHeaders("Link");
if ( links != null ) {
for ( Header h : headResponse.getHeaders("Link") ) {
if (links != null) {
for (Header h : headResponse.getHeaders("Link")) {
final Link link = Link.valueOf(h.getValue());
if (link.getRel().equals("describedby")) {
descriptionURI = link.getUri();
Expand All @@ -121,10 +123,14 @@ public NamedFields get() {
return getNamedFields(rdf, uri);
} catch (IOException | HttpException e) {
throw propagate(e);
} finally {
headRequest.releaseConnection();
}
}

private NamedFields getNamedFields(final Model rdf, final URI uri) throws IOException, HttpException {
HttpGet transformedResourceRequest = null;

final NodeIterator nodeIterator =
rdf.listObjectsOfProperty(createResource(uri.toString()),
INDEXING_TRANSFORM_PREDICATE);
Expand All @@ -136,21 +142,27 @@ private NamedFields getNamedFields(final Model rdf, final URI uri) throws IOExce
final String transformKey =
indexingTransform.asLiteral().getString();
LOGGER.debug("Discovered transform key: {}", transformKey);
final HttpGet transformedResourceRequest =
transformedResourceRequest =
new HttpGet(uri.toString() + "/fcr:transform/" + transformKey);
LOGGER.debug("Retrieving transformed resource from: {}",
transformedResourceRequest.getURI());
try {
LOGGER.debug("Retrieving transformed resource from: {}",
transformedResourceRequest.getURI());

final HttpResponse response =
httpClient.execute(transformedResourceRequest);
if (response.getStatusLine().getStatusCode() != SC_OK) {
throw new HttpException(response.getStatusLine().toString());
}
try (
Reader r =
new InputStreamReader(response.getEntity().getContent(),
"UTF8")) {
return gson.fromJson(r, typeToken);
final HttpResponse response =
httpClient.execute(transformedResourceRequest);
if (response.getStatusLine().getStatusCode() != SC_OK) {
throw new HttpException(response.getStatusLine().toString());
}
try (
Reader r =
new InputStreamReader(response.getEntity().getContent(),
"UTF8")) {
return gson.fromJson(r, typeToken);
}
} finally {
if (transformedResourceRequest != null) {
transformedResourceRequest.releaseConnection();
}
}
}

Expand Down
28 changes: 17 additions & 11 deletions fcrepo-message-consumer-core/src/main/java/org/fcrepo/indexer/RdfRetriever.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpUriRequest;
import org.slf4j.Logger;

import com.google.common.base.Supplier;
Expand All @@ -60,7 +59,7 @@ public class RdfRetriever implements Supplier<Model> {

/**
* @param identifier the URI identifier
* @param client the http client
* @param client the http client
*/
public RdfRetriever(final URI identifier, final HttpClient client) {
this.identifier = identifier;
Expand All @@ -70,41 +69,48 @@ public RdfRetriever(final URI identifier, final HttpClient client) {
@Override
public Model get() {

final HttpHead headRequest = new HttpHead(identifier);
HttpGet request = null;

try {
// make an initial HEAD request and check Link headers for descriptions located elsewhere
final HttpHead headRequest = new HttpHead(identifier);
final HttpResponse headResponse = httpClient.execute(headRequest);
URI descriptionURI = null;
final Header[] links = headResponse.getHeaders("Link");
if ( links != null ) {
for ( Header h : headResponse.getHeaders("Link") ) {
if (links != null) {
for (Header h : headResponse.getHeaders("Link")) {
final Link link = Link.valueOf(h.getValue());
if ( link.getRel().equals("describedby") ) {
if (link.getRel().equals("describedby")) {
descriptionURI = link.getUri();
LOGGER.debug("Using URI from Link header: {}", descriptionURI);
}
}
}
if ( descriptionURI == null ) {
if (descriptionURI == null) {
descriptionURI = identifier;
}

final HttpUriRequest request = new HttpGet(descriptionURI);
request = new HttpGet(descriptionURI);
request.addHeader("Accept", RDF_SERIALIZATION);
LOGGER.debug("Retrieving RDF content from: {}...", request.getURI());
final HttpResponse response = httpClient.execute(request);
if (response.getStatusLine().getStatusCode() == SC_OK) {
try (
Reader r =
new InputStreamReader(
response.getEntity().getContent(), "UTF8")) {
Reader r =
new InputStreamReader(
response.getEntity().getContent(), "UTF8")) {
return createDefaultModel().read(r, "", "N3");
}
} else {
throw new HttpException(response.getStatusLine().toString());
}
} catch (IOException | HttpException e) {
throw propagate(e);
} finally {
headRequest.releaseConnection();
if (request != null) {
request.releaseConnection();
}
}
}

Expand Down