From 1872426d64c065207e465eded986634581332919 Mon Sep 17 00:00:00 2001 From: ljuillerat Date: Tue, 8 Aug 2023 14:30:11 +0200 Subject: [PATCH 1/4] Add transport parameters transport.vfs.MinimumAge and transport.vfs.MaximumAge which check file age (seconds) --- .../synapse/commons/vfs/VFSConstants.java | 5 +++ .../synapse/transport/vfs/PollTableEntry.java | 34 +++++++++++++++++++ .../transport/vfs/VFSTransportListener.java | 18 +++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/modules/commons/src/main/java/org/apache/synapse/commons/vfs/VFSConstants.java b/modules/commons/src/main/java/org/apache/synapse/commons/vfs/VFSConstants.java index 764162f69e..8f4b6f72ed 100644 --- a/modules/commons/src/main/java/org/apache/synapse/commons/vfs/VFSConstants.java +++ b/modules/commons/src/main/java/org/apache/synapse/commons/vfs/VFSConstants.java @@ -200,6 +200,11 @@ public final class VFSConstants { public static final String SCHEME_FTPS = "ftps"; // sftp scheme file option list public static enum SFTP_FILE_OPTION {Identities, UserDirIsRoot, IdentityPassPhrase}; + + /** Parameter for minimum age **/ + public static final String TRANSPORT_FILE_MINIMUM_AGE = "transport.vfs.MinimumAge"; + /** Parameter for maximum age **/ + public static final String TRANSPORT_FILE_MAXIMUM_AGE = "transport.vfs.MaximumAge"; public static final String FILE_TYPE_PREFIX = "transport.vfs.fileType"; public static final String FILE_TYPE = "filetype"; diff --git a/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/PollTableEntry.java b/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/PollTableEntry.java index 90dd5917f6..142fd5de78 100644 --- a/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/PollTableEntry.java +++ b/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/PollTableEntry.java @@ -143,6 +143,10 @@ public class PollTableEntry extends AbstractPollTableEntry { private static final Log log = LogFactory.getLog(PollTableEntry.class); + private Long minimumAge = null; //defines a minimum age of a file before being consumed. Use to avoid just written files to be consumed + private Long maximumAge = null; //defines a maximum age of a file being consumed. Old files will stay in the directory + + public PollTableEntry(boolean fileLocking) { this.fileLocking = fileLocking; } @@ -499,6 +503,18 @@ public void setSubfolderTimestamp(String subfolderTimestamp) { this.subfolderTimestamp = subfolderTimestamp; } + public Long getMinimumAge() { + return minimumAge; + } + + public Long getMaximumAge() { + return maximumAge; + } + + public boolean hasAgeCheck(){ + return minimumAge != null||maximumAge != null; + } + @Override public boolean loadConfiguration(ParameterInclude params) throws AxisFault { @@ -515,6 +531,7 @@ public boolean loadConfiguration(ParameterInclude params) throws AxisFault { } protected boolean loadConfigurationsFromService(ParameterInclude params) throws AxisFault { + System.out.println("loading config from service"); fileURI = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_FILE_URI); if (fileURI == null) { log.warn("transport.vfs.FileURI parameter is missing in the proxy service configuration"); @@ -690,6 +707,23 @@ protected boolean loadConfigurationsFromService(ParameterInclude params) throws } } + String strMinimumAge = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_MINIMUM_AGE); + if(strMinimumAge != null){ + try { + minimumAge = Long.parseLong(strMinimumAge); + } catch (NumberFormatException nfe) { + log.warn("VFS File MinimumAge value is invalid : " + strMinimumAge, nfe); + } + } + String strMaximumAge = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_MAXIMUM_AGE); + if(strMaximumAge != null){ + try { + maximumAge = Long.parseLong(strMaximumAge); + } catch (NumberFormatException nfe) { + log.warn("VFS File MaximumAge value is invalid : " + strMinimumAge, nfe); + } + } + String strAutoLock = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_AUTO_LOCK_RELEASE); autoLockRelease = false; diff --git a/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/VFSTransportListener.java b/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/VFSTransportListener.java index 073fa959fe..4371216bce 100644 --- a/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/VFSTransportListener.java +++ b/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/VFSTransportListener.java @@ -37,7 +37,7 @@ import org.apache.axis2.transport.base.BaseConstants; import org.apache.axis2.transport.base.BaseUtils; import org.apache.axis2.transport.base.ManagementSupport; -import org.apache.axis2.transport.base.threads.WorkerPool; +import org.apache.axis2.transport.base.threads.WorkerPool; import org.apache.commons.io.FileUtils; import org.apache.commons.io.input.AutoCloseInputStream; import org.apache.commons.lang3.StringUtils; @@ -452,6 +452,22 @@ protected void scanFileOrDirectory(final PollTableEntry entry, String fileURI) { isFailedRecord = isFailedRecord(child, entry); } + if(entry.getMinimumAge() != null){ + long age = child.getContent().getLastModifiedTime(); + long time = System.currentTimeMillis(); + if((time-age)/1000 <= entry.getMinimumAge()){ + continue; + } + } + + if(entry.getMaximumAge() != null){ + long age = child.getContent().getLastModifiedTime(); + long time = System.currentTimeMillis(); + if((time-age)/1000 >= entry.getMaximumAge()){ + continue; + } + } + if(entry.getFileNamePattern()!=null && child.getName().getBaseName().matches(entry.getFileNamePattern())){ //child's file name matches the file name pattern From cbeb42ed46c4ee5de989dd7a1ba45949c3db9601 Mon Sep 17 00:00:00 2001 From: ljuillerat <33685946+ljuillerat@users.noreply.github.com> Date: Wed, 21 Aug 2024 09:36:00 +0200 Subject: [PATCH 2/4] Remove unnecessary System.out occurrence --- .../java/org/apache/synapse/transport/vfs/PollTableEntry.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/PollTableEntry.java b/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/PollTableEntry.java index 142fd5de78..03840c7999 100644 --- a/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/PollTableEntry.java +++ b/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/PollTableEntry.java @@ -531,7 +531,6 @@ public boolean loadConfiguration(ParameterInclude params) throws AxisFault { } protected boolean loadConfigurationsFromService(ParameterInclude params) throws AxisFault { - System.out.println("loading config from service"); fileURI = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_FILE_URI); if (fileURI == null) { log.warn("transport.vfs.FileURI parameter is missing in the proxy service configuration"); From 2fb9de50fa26abc3cc415f3d9e35164e084ae90d Mon Sep 17 00:00:00 2001 From: ljuillerat <33685946+ljuillerat@users.noreply.github.com> Date: Wed, 21 Aug 2024 09:38:28 +0200 Subject: [PATCH 3/4] Cleanup --- .../org/apache/synapse/transport/vfs/VFSTransportListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/VFSTransportListener.java b/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/VFSTransportListener.java index 4371216bce..021a1cadfd 100644 --- a/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/VFSTransportListener.java +++ b/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/VFSTransportListener.java @@ -37,7 +37,7 @@ import org.apache.axis2.transport.base.BaseConstants; import org.apache.axis2.transport.base.BaseUtils; import org.apache.axis2.transport.base.ManagementSupport; -import org.apache.axis2.transport.base.threads.WorkerPool; +import org.apache.axis2.transport.base.threads.WorkerPool; import org.apache.commons.io.FileUtils; import org.apache.commons.io.input.AutoCloseInputStream; import org.apache.commons.lang3.StringUtils; From 4d0a0f14408b38857ff3fc05921c2cae3045aca7 Mon Sep 17 00:00:00 2001 From: ljuillerat <33685946+ljuillerat@users.noreply.github.com> Date: Tue, 3 Sep 2024 08:39:07 +0200 Subject: [PATCH 4/4] Update PollTableEntry.java Remove unreferenced method --- .../java/org/apache/synapse/transport/vfs/PollTableEntry.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/PollTableEntry.java b/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/PollTableEntry.java index 03840c7999..04be8240eb 100644 --- a/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/PollTableEntry.java +++ b/modules/transports/core/vfs/src/main/java/org/apache/synapse/transport/vfs/PollTableEntry.java @@ -511,10 +511,6 @@ public Long getMaximumAge() { return maximumAge; } - public boolean hasAgeCheck(){ - return minimumAge != null||maximumAge != null; - } - @Override public boolean loadConfiguration(ParameterInclude params) throws AxisFault {