diff --git a/Cargo.toml b/Cargo.toml index 16b03e5..1b43ab0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ compact_str = "0.7.1" napi = { version = "2.14.1", default-features = false, features = ["napi4", "async", "tokio_rt"] } napi-derive = "2.14.2" num_cpus = "1.16.0" -spider = { version = "1.50.8", features = ["napi", "budget", "cron", "regex", "cookies"] } +spider = { version = "1.50.8", features = ["napi", "budget", "cron", "regex", "cookies", "socks"] } [target.x86_64-unknown-linux-gnu.dependencies] openssl-sys = { version = "0.9.96", features = ["vendored"] } diff --git a/book/src/cron-job.md b/book/src/cron-job.md index 746485a..7a84a87 100644 --- a/book/src/cron-job.md +++ b/book/src/cron-job.md @@ -3,14 +3,14 @@ Use a cron job that can run any time of day to gather website data. ```ts -import { Website, type NPage } from "@spider-rs/spider-rs"; +import { Website } from "@spider-rs/spider-rs"; const website = new Website("https://choosealicense.com") .withCron("1/5 * * * * *") .build(); -const onPageEvent = (err: Error | null, value: NPage) => { - links.push(value); +const onPageEvent = (err, value) => { + console.log(value); }; const handle = await website.runCron(onPageEvent); diff --git a/book/src/website.md b/book/src/website.md index 8f700e2..59cc98e 100644 --- a/book/src/website.md +++ b/book/src/website.md @@ -49,3 +49,105 @@ const website = new Website("https://choosealicense.com") ``` View the [cron](./cron-job.md) section for details how to use the cron. + +### Budget + +Add a crawl budget that prevents crawling `x` amount of pages. + +```ts +const website = new Website("https://choosealicense.com") + .withBudget({ + "*": 1, + }) + .build(); +``` + +### Subdomains + +Include subdomains in request. + +```ts +const website = new Website("https://choosealicense.com") + .withSubdomains(true) + .build(); +``` + +### TLD + +Include TLDs in request. + +```ts +const website = new Website("https://choosealicense.com") + .withTlds(true) + .build(); +``` + +### External Domains + +Add external domains to include with the website. + +```ts +const website = new Website("https://choosealicense.com") + .withExternalDomains(["https://www.myotherdomain.com"]) + .build(); +``` + +### Proxy + +Use a proxy to crawl a website. + +```ts +const website = new Website("https://choosealicense.com") + .withProxies(["https://www.myproxy.com"]) + .build(); +``` + +### Delays + +Add delays between pages. + +```ts +const website = new Website("https://choosealicense.com") + .withDelays(200) + .build(); +``` + +### User-Agent + +Use a custom User-Agent. + +```ts +const website = new Website("https://choosealicense.com") + .withUserAgent("mybot/v1") + .build(); +``` + +### Request Timeout + +Add a request timeout per page in miliseconds. Example shows 30 seconds. + +```ts +const website = new Website("https://choosealicense.com") + .withRequestTimeout(30000) + .build(); +``` + +### Respect Robots + +Respect the robots.txt file. + +```ts +const website = new Website("https://choosealicense.com") + .withRespectRobotsTxt(true) + .build(); +``` + +### Http2 Prior Knowledge + +Use http2 to connect if you know the website servers supports this. + +```ts +const website = new Website("https://choosealicense.com") + .withHttp2PriorKnowledge(true) + .build(); +```