Skip to content

Commit

Permalink
.NET Core 3.0, use base SFTP connection instead of SCP for xfers
Browse files Browse the repository at this point in the history
  • Loading branch information
@ScottRFrost
ScottRFrost committed Dec 4, 2019
1 parent 2f89310 commit cb22496
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 65 deletions.
11 changes: 4 additions & 7 deletions src/SimpleSFTPSyncCore/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,6 @@ public static void Main(string[] args)
}
}
}


// Process MKVs
if (mkvs.Count > 0)
Expand Down Expand Up @@ -159,10 +158,10 @@ public static void Main(string[] args)
// Direct SQL command
else if (args[0] == "sql")
{
var db = new SimpleSFTPSyncCoreContext();
using var db = new SimpleSFTPSyncCoreContext();
var command = string.Join(" ", args).Substring(4);
#pragma warning disable EF1000 // Possible SQL injection vulnerability.
Log(db.Database.ExecuteSqlCommand(command) + " rows affected");
Log(db.Database.ExecuteSqlRaw(command) + " rows affected");
#pragma warning restore EF1000 // Possible SQL injection vulnerability.
}

Expand Down Expand Up @@ -205,10 +204,8 @@ public static void Log(string logText)
var logBytes = new UTF8Encoding(true).GetBytes(DateTime.Now.ToString("HH:mm:ss") + " " + logText + "\r\n");
lock (logLock)
{
using (FileStream log = new FileStream(logPath, FileMode.Append, FileAccess.Write))
{
log.Write(logBytes, 0, logBytes.Length);
}
using FileStream log = new FileStream(logPath, FileMode.Append, FileAccess.Write);
log.Write(logBytes, 0, logBytes.Length);
}
}
}
Expand Down
104 changes: 54 additions & 50 deletions src/SimpleSFTPSyncCore/SimpleSFTPSync.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public void StartRun()
sftp.Connect();
Log("Connected to " + config.hostname + " running " + sftp.ConnectionInfo.ServerVersion);
Log("Checking for new files to download...");
foreach(var remoteDir in config.remoteDir)
foreach (var remoteDir in config.remoteDir)
{
Log("Checking for new files in " + remoteDir);
var foundFiles = ListFilesRecursive(sftp, remoteDir, string.Empty).ToString();
Expand All @@ -80,7 +80,8 @@ public void StartRun()
{
Log(syncFile.RemotePath + " and " + localPath + " are the same size. Skipping.");
syncFile.DateDownloaded = DateTime.Now.ToString();
lock(dbLock) {
lock (dbLock)
{
db.SaveChanges();
}
continue;
Expand All @@ -104,33 +105,39 @@ public void StartRun()
{
var startTime = DateTime.Now;

// ASync SFTP
//using (var fileStream = File.OpenWrite(localPath))
//{
//var download = (SftpDownloadAsyncResult)sftp.BeginDownloadFile(syncFile.RemotePath, fileStream);
//ulong lastDownloadedBytes = 0;
//while (!download.IsCompleted)
//{
// Status(string.Format("Downloaded {0:n0} / {1:n0} KB @ {2:n0} KB/sec", download.DownloadedBytes / 1024, syncFile.Length / 1024, (download.DownloadedBytes - lastDownloadedBytes) / 1024));
// lastDownloadedBytes = download.DownloadedBytes;
// Thread.Sleep(1000);
//}
//}

// Sync SFTP
using (var scp = new ScpClient(config.hostname, config.port, config.username, config.password))
{
scp.Connect();
scp.Downloading += Scp_Downloading;
scp.Download(syncFile.RemotePath, new DirectoryInfo(localDirectory));
scp.Disconnect();
}
// ASync SFTP
//using (var fileStream = File.OpenWrite(localPath))
//{
// var download = (SftpDownloadAsyncResult)sftp.BeginDownloadFile(syncFile.RemotePath, fileStream);
// ulong lastDownloadedBytes = 0;
// while (!download.IsCompleted)
// {
// Status(string.Format("Downloaded {0:n0} / {1:n0} KB @ {2:n0} KB/sec", download.DownloadedBytes / 1024, syncFile.Length / 1024, (download.DownloadedBytes - lastDownloadedBytes) / 1024));
// lastDownloadedBytes = download.DownloadedBytes;
// Thread.Sleep(1000);
// }
//}

var endTime = DateTime.Now;
// Sync SFTP
using (var fileStream = File.OpenWrite(localPath))
{
sftp.DownloadFile(syncFile.RemotePath, fileStream);
}

var timespan = TimeSpan.FromSeconds((endTime - startTime).TotalSeconds);
Log(string.Format("Downloaded Successfully at {0:n0} KB/sec", (syncFile.Length / 1024) / timespan.TotalSeconds));
success = true;
////// Sync SCP
////using (var scp = new ScpClient(config.hostname, config.port, config.username, config.password))
////{
//// scp.Connect();
//// scp.Downloading += Scp_Downloading;
//// scp.Download(syncFile.RemotePath, new DirectoryInfo(localDirectory));
//// scp.Disconnect();
////}

var endTime = DateTime.Now;

var timespan = TimeSpan.FromSeconds((endTime - startTime).TotalSeconds);
Log(string.Format("Downloaded Successfully at {0:n0} KB/sec", syncFile.Length / 1024 / timespan.TotalSeconds));
success = true;
}
catch (Exception exception)
{
Expand All @@ -142,12 +149,12 @@ public void StartRun()
{

syncFile.DateDownloaded = DateTime.Now.ToString();
lock(dbLock)
lock (dbLock)
{
db.SaveChanges();
}

if (localPath.EndsWith(".part1.rar", StringComparison.Ordinal) || !localPath.Contains(".part") && localPath.EndsWith(".rar", StringComparison.Ordinal))
if (localPath.EndsWith(".part1.rar", StringComparison.Ordinal) || (!localPath.Contains(".part") && localPath.EndsWith(".rar", StringComparison.Ordinal)))
{
rars.Add(localPath);
Log("Added " + localPath + " to auto-unrar queue");
Expand All @@ -162,7 +169,7 @@ public void StartRun()
{
Log(syncFile.RemotePath + " no longer exists");
syncFile.DateDownloaded = DateTime.Now.ToString();
lock(dbLock)
lock (dbLock)
{
db.SaveChanges();
}
Expand All @@ -173,15 +180,14 @@ public void StartRun()
Log("!!ERROR!! while downloading and scanning " + syncFile.RemotePath + " - " + exception);
}
}
lock(dbLock)
lock (dbLock)
{
db.SaveChanges();
}

sftp.Disconnect();
}


Log("Downloading complete. Processing " + rars.Count + " rars and " + mkvs.Count + " mkvs ...");
// Unrar
foreach (var rar in rars)
Expand Down Expand Up @@ -212,7 +218,7 @@ public void StartRun()

// MKV move & rename
MoveFiles(mkvs);

Log("All jobs complete. Closing in 60 seconds...");
Thread.Sleep(60000);
}
Expand Down Expand Up @@ -245,7 +251,7 @@ public void MoveFiles(List<string> mkvs, bool CopyInsteadOfMove = false)
{
Log("Moving TV " + mkv + " -->\r\n " + filePath);
}

if (filename.Contains(Path.DirectorySeparatorChar))
{
Directory.CreateDirectory(config.tvDir + Path.DirectorySeparatorChar + filename.Substring(0, filename.LastIndexOf(Path.DirectorySeparatorChar)));
Expand Down Expand Up @@ -287,7 +293,7 @@ public void MoveFiles(List<string> mkvs, bool CopyInsteadOfMove = false)
File.Move(mkv, filePath);
Log("Moved Successfully");
}

}
}
else
Expand All @@ -302,7 +308,7 @@ public void MoveFiles(List<string> mkvs, bool CopyInsteadOfMove = false)
{
Log("Moving Movie " + mkv + " -->\r\n " + filePath);
}

if (filename.Contains(Path.DirectorySeparatorChar))
{
Directory.CreateDirectory(config.movieDir + Path.DirectorySeparatorChar + filename.Substring(0, filename.LastIndexOf(Path.DirectorySeparatorChar)));
Expand Down Expand Up @@ -358,11 +364,11 @@ public void MoveFiles(List<string> mkvs, bool CopyInsteadOfMove = false)
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void Scp_Downloading(object sender, Renci.SshNet.Common.ScpDownloadEventArgs e)
{
var percentage = ((double)e.Downloaded / (double)e.Size) * 100D;
Status(string.Format("{0} - {1:n2}% {2:n0} / {3:n0}", e.Filename, percentage, e.Downloaded, e.Size));
}
////private void Scp_Downloading(object sender, Renci.SshNet.Common.ScpDownloadEventArgs e)
////{
//// var percentage = ((double)e.Downloaded / (double)e.Size) * 100D;
//// Status(string.Format("{0} - {1:n2}% {2:n0} / {3:n0}", e.Filename, percentage, e.Downloaded, e.Size));
////}

/// <summary>
/// Scan entire directory structure of remote SFTP server
Expand Down Expand Up @@ -394,7 +400,7 @@ private int ListFilesRecursive(SftpClient sftp, string basePath, string subDirec
if (file == null)
{
Log("Found New file: " + basePath + filePath);
lock(dbLock)
lock (dbLock)
{
db.SyncFile.Add(new SyncFile
{
Expand All @@ -406,7 +412,7 @@ private int ListFilesRecursive(SftpClient sftp, string basePath, string subDirec
});
db.SaveChanges();
}

foundFiles++;
}
else if (file.Length != sftpFile.Length || Convert.ToDateTime(file.RemoteDateModified) != sftpFile.LastWriteTime)
Expand All @@ -415,7 +421,7 @@ private int ListFilesRecursive(SftpClient sftp, string basePath, string subDirec
file.DateDownloaded = null;
file.Length = sftpFile.Length;
file.RemoteDateModified = sftpFile.LastWriteTime.ToString();
lock(dbLock)
lock (dbLock)
{
db.SaveChanges();
}
Expand Down Expand Up @@ -448,12 +454,10 @@ public void Log(string logText)
}
Console.WriteLine(DateTime.Now.ToString("HH:mm:ss") + " " + logText);
var logBytes = new UTF8Encoding(true).GetBytes(DateTime.Now.ToString("HH:mm:ss") + " " + logText + "\r\n");
lock(logLock)
lock (logLock)
{
using(FileStream log = new FileStream(logPath, FileMode.Append, FileAccess.Write))
{
log.Write(logBytes, 0, logBytes.Length);
}
using FileStream log = new FileStream(logPath, FileMode.Append, FileAccess.Write);
log.Write(logBytes, 0, logBytes.Length);
}
}

Expand Down
18 changes: 10 additions & 8 deletions src/SimpleSFTPSyncCore/SimpleSFTPSyncCore.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<VersionPrefix>2.2.3</VersionPrefix>
<TargetFramework>netcoreapp2.2</TargetFramework>
<VersionPrefix>3.0.1</VersionPrefix>
<TargetFramework>netcoreapp3.0</TargetFramework>
<AssemblyName>SimpleSFTPSyncCore</AssemblyName>
<OutputType>Exe</OutputType>
<PackageId>SimpleSFTPSyncCore</PackageId>
<RuntimeIdentifiers>win10-x64</RuntimeIdentifiers>
<RuntimeFrameworkVersion>2.2.3</RuntimeFrameworkVersion>
<PublishTrimmed>true</PublishTrimmed>
<RuntimeFrameworkVersion>3.0.1</RuntimeFrameworkVersion>
<GenerateAssemblyConfigurationAttribute>false</GenerateAssemblyConfigurationAttribute>
<GenerateAssemblyCompanyAttribute>false</GenerateAssemblyCompanyAttribute>
<GenerateAssemblyProductAttribute>false</GenerateAssemblyProductAttribute>
<Version>2.2.3</Version>
<Version>3.0.1</Version>
</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
Expand All @@ -27,14 +28,15 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="2.2.6" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="2.2.6" />
<FrameworkReference Include="Microsoft.AspNetCore.App" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="3.1.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="3.1.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite.Design" Version="1.1.6" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="2.2.6">
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="3.1.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
</PackageReference>
<PackageReference Include="Newtonsoft.Json" Version="12.0.2" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="SSH.NET" Version="2016.1.0" />
</ItemGroup>
</Project>

0 comments on commit cb22496

Please sign in to comment.