Skip to content

Commit

Permalink
Fix gelbooru downloading & clean code a little bit
Browse files Browse the repository at this point in the history 
Also tried to fix danbooru downloading. I am not done with it, but I'm
near to finishing it.
  • Loading branch information
@fgRuslan
fgRuslan committed Mar 9, 2023
1 parent 273517c commit 3f93291
Show file tree
Hide file tree
Showing 11 changed files with 180 additions and 190 deletions.
6 changes: 3 additions & 3 deletions App.config
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8" ?>
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.1"/>
</startup>
</configuration>
</configuration>
6 changes: 4 additions & 2 deletions BooruDownloader.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
Expand All @@ -9,7 +9,7 @@
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>BooruDownloader</RootNamespace>
<AssemblyName>BooruDownloader</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<PublishUrl>опубликовать\</PublishUrl>
<Install>true</Install>
Expand All @@ -26,6 +26,7 @@
<IsWebBootstrapper>false</IsWebBootstrapper>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
Expand Down Expand Up @@ -85,6 +86,7 @@
<Compile Include="Properties\Resources.Designer.cs">
<AutoGen>True</AutoGen>
<DependentUpon>Resources.resx</DependentUpon>
<DesignTime>True</DesignTime>
</Compile>
<None Include="Properties\Settings.settings">
<Generator>SettingsSingleFileGenerator</Generator>
Expand Down
100 changes: 51 additions & 49 deletions DanEngine.cs
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Net;
using Microsoft.Win32;
using System.Runtime.InteropServices;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using System.Windows.Forms;
using System.Xml;

namespace BooruDownloader
{
Expand All @@ -20,37 +15,37 @@ public override type getType()
return type.DAN;
}

public override string ExtFromURL(string line)
public override string ExtensionFromUrl(string line)
{
var ext = "";
var extension = "";
var match = Regex.Match(line, "(?:)\\.[\\d\\w]+$", RegexOptions.Compiled);
if (match.Success)
ext = match.Value;
return ext;
extension = match.Value;
return extension;
}
public override string FnameFromURL(string line)
public override string FilenameFromUrl(string line)
{
var fname = "";
var filename = "";
var match = Regex.Match(line, "(?:)[\\d\\w]+\\.[\\d\\w]+$", RegexOptions.Compiled);
if (match.Success)
fname = match.Value;
return fname;
filename = match.Value;
return filename;
}

public override string Truncate(string value, int maxChars)
{
return value.Length <= maxChars ? value : value.Substring(0, maxChars) + "...";
}

public override async void downloadImage(string url, string tags, bool keepOriginalNames, string rating)
public override async void DownloadImage(string url, string tags, bool keepOriginalNames, string rating)
{
//I don't know what is this shit!
string fullpath = "./out/" + rating + tags + ExtFromURL(url);
string fullpath = "./out/" + rating + tags + ExtensionFromUrl(url);
string regexSearch = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars());
Regex r = new Regex(string.Format("[{0}]", Regex.Escape(regexSearch)));
fullpath = r.Replace(fullpath, "");

string shortPath = Path.GetFullPath("./out/" + rating + ExtFromURL(url));
string shortPath = Path.GetFullPath("./out/" + rating + ExtensionFromUrl(url));
string extension = fullpath.Substring(fullpath.Length - 5);
if (fullpath.Length > 259)
fullpath = Truncate(fullpath, 259 - shortPath.Length - 4);
Expand All @@ -63,85 +58,92 @@ public override async void downloadImage(string url, string tags, bool keepOrigi
try
{
if (keepOriginalNames)
wc.DownloadFileAsync(new System.Uri(url), "./out/" + rating + FnameFromURL(url));
wc.DownloadFileAsync(new System.Uri(url), "./out/" + rating + FilenameFromUrl(url));
else
await wc.DownloadFileTaskAsync(new System.Uri(url), "./out/" + fullpath + extension);
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString(), ex.GetType().ToString());
//wc.DownloadFileAsync(new System.Uri(url), "./out/" + rating + FnameFromURL(url));
wc.DownloadFileAsync(new System.Uri(url), "./out/" + rating + FilenameFromUrl(url));
}
}
}

public override string downloadPosts(string domain, string tags, int page, bool keepOriginalNames, bool includeRating)
public override string DownloadPosts(string domain, string tags, int page, bool keepOriginalNames, bool includeRating)
{
ServicePointManager.DefaultConnectionLimit = 9999;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;

HttpWebRequest request = (HttpWebRequest)WebRequest.Create(domain + "/posts.xml?page=dapi&s=post&q=index&limit=1&tags=" + tags + "&page=" + page);
string apiKey = "";
string login = "";

HttpWebRequest request = (HttpWebRequest)WebRequest.Create(domain + "/posts.xml?page=dapi&s=post&q=index&limit=1&tags=" + tags + "&page=" + page + "&api_key=" + apiKey + "&login=" + login);
request.UserAgent = ".NET Framework Test Client";
request.Accept = "text/xml";
//request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0";

HttpWebResponse response = (HttpWebResponse)request.GetResponse();
using (Stream stream = response.GetResponseStream())
{
StreamReader reader = new StreamReader(stream, Encoding.UTF8);
String responseString = reader.ReadToEnd();
string responseString = reader.ReadToEnd();

doc.LoadXml(responseString);
root = doc.DocumentElement;
XmlNode node = (XmlNode)doc.DocumentElement;
document.LoadXml(responseString);
root = document.DocumentElement;
XmlNode node = document.DocumentElement;
XmlNode sourceNode = node.SelectSingleNode("post/file-url");
XmlNode tagsNode = node.SelectSingleNode("post/tag-string");
XmlNode ratingNode = node.SelectSingleNode("post/rating");

if (sourceNode == null)
{
return "";
}

url = sourceNode.InnerXml;
var tagstring = tagsNode.InnerXml;
var postTags = tagsNode.InnerXml;
var rating = ratingNode.InnerXml;
var ratingstr = "";

if (includeRating)
{
if (rating == "q")
ratingstr = "questionable ";
if (rating == "e")
ratingstr = "nsfw ";
rating = "questionable ";
if (rating == "e" || rating == "explicit")
rating = "nsfw ";
if (rating == "s")
ratingstr = "safe ";
rating = "safe ";
}
Console.WriteLine(url);
downloadImage(url, tagstring, keepOriginalNames, ratingstr);
DownloadImage(url, postTags, keepOriginalNames, rating);
}
return url;
}

public override int getPostCount(string domain, string tags)
public override int GetPostCount(string domain, string tags)
{

ServicePointManager.DefaultConnectionLimit = 9999;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;

HttpWebRequest request = (HttpWebRequest)WebRequest.Create(domain + "/posts.xml?page=dapi&s=post&q=index&tags=" + tags);
request.Credentials = CredentialCache.DefaultCredentials;
//request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)";
string apiKey = "";
string login = "";

HttpWebRequest request = (HttpWebRequest)WebRequest.Create(domain + "/posts.xml?page=dapi&s=post&q=index&tags=" + tags + "&api_key=" + apiKey + "&login=" + login);
request.UserAgent = ".NET Framework Test Client";
request.Accept = "text/xml";
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
#if DEBUG
AllocConsole();
#endif
using (Stream stream = response.GetResponseStream())
{
StreamReader reader = new StreamReader(stream, Encoding.UTF8);
String responseString = reader.ReadToEnd();
doc.LoadXml(responseString);
root = doc.DocumentElement;
string responseString = reader.ReadToEnd();
document.LoadXml(responseString);
root = document.DocumentElement;

XmlNodeList elemList = root.GetElementsByTagName("post");
int count = 0;
for (int i = 0; i < elemList.Count; i++)
{
count++;
}
return Convert.ToInt32(count);
XmlNodeList elementList = root.GetElementsByTagName("post");
return Convert.ToInt32(elementList.Count);
}
}
}
Expand Down
1 change: 0 additions & 1 deletion Detector.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace BooruDownloader
{
Expand Down
20 changes: 8 additions & 12 deletions EngineBase.cs
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using System.Runtime.InteropServices;
using System.Xml;

namespace BooruDownloader
{
public abstract class EngineBase
{
public XmlDocument doc = new XmlDocument();
public XmlDocument document = new XmlDocument();
public XmlElement root;
public XmlElement post;
public string url;
public string rating;

Expand All @@ -24,12 +20,12 @@ public enum type
[DllImport("kernel32.dll", SetLastError = true)]
public static extern bool AllocConsole();

public abstract string ExtFromURL(string line);
public abstract string FnameFromURL(string line);
public abstract string ExtensionFromUrl(string line);
public abstract string FilenameFromUrl(string line);
public abstract string Truncate(string line, int mChar);
public abstract void downloadImage(string url, string tags, bool keepOrigName, string rating);
public abstract string downloadPosts(string url, string tags, int page, bool keepOrigName, bool inclRating);
public abstract int getPostCount(string domain, string tags);
public abstract void DownloadImage(string url, string tags, bool keepOrigName, string rating);
public abstract string DownloadPosts(string url, string tags, int page, bool keepOrigName, bool inclRating);
public abstract int GetPostCount(string domain, string tags);
public abstract type getType();
}
}
16 changes: 3 additions & 13 deletions Form1.cs
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,8 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Net;
using Microsoft.Win32;
using System.Runtime.InteropServices;
using System.IO;
using System.Text.RegularExpressions;
using System.Xml;

namespace BooruDownloader
{
Expand Down Expand Up @@ -83,7 +73,7 @@ private async void downloadButton_Click(object sender, EventArgs e)
else
engine = new GelEngine();
}
int postCount = engine.getPostCount(domainBox.Text, tagsBox.Text);
int postCount = engine.GetPostCount(domainBox.Text, tagsBox.Text);
Console.WriteLine(postCount);
if (postCount == 0)
{
Expand All @@ -110,7 +100,7 @@ private async void downloadButton_Click(object sender, EventArgs e)
{
if (alreadyDownloaded <= limitBoxText)
{
await Task.Run(() => engine.downloadPosts(domainBox.Text, tagsBox.Text, i, checkBox1.Checked, ratingCheckBox.Checked));
await Task.Run(() => engine.DownloadPosts(domainBox.Text, tagsBox.Text, i, checkBox1.Checked, ratingCheckBox.Checked));
label4.Text = Convert.ToString(postCount - i) + " left";
alreadyDownloaded++;
}
Expand Down
Loading

0 comments on commit 3f93291

Please sign in to comment.