diff --git a/book/src/README.md b/book/src/README.md index 87d32dc..f3192f4 100644 --- a/book/src/README.md +++ b/book/src/README.md @@ -1,11 +1,9 @@ # Introduction -Spider-RS is the fastest web crawler and indexer written in Rust. +Spider-RS is the fastest web crawler and indexer written in Rust ported to Node.js. Spider powers some big tools and helps bring the crawling aspect to almost no downtime with the correct setup, view the [spider](https://github.com/spider-rs/spider) project to learn more. -Node.js powerhouse crawling in a few lines! - ```ts import { Website } from "@spider-rs/spider-rs"; diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index e6d2556..5a5f37e 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -10,7 +10,7 @@ # Configuration - [Website](./website.md) -- [PAge](./page.md) +- [Page](./page.md) # Usage diff --git a/book/src/page.md b/book/src/page.md index 1f3f36a..c436899 100644 --- a/book/src/page.md +++ b/book/src/page.md @@ -1,6 +1,6 @@ # Page -A single page on a website, useful if you just need one or the root url. +A single page on a website, useful if you just the root url. ## New Page diff --git a/book/src/simple.md b/book/src/simple.md index 6e3e38e..4732501 100644 --- a/book/src/simple.md +++ b/book/src/simple.md @@ -15,16 +15,7 @@ A basic example. ```ts import { Website } from "@spider-rs/spider-rs"; -const website = new Website("https://rsseau.fr") - .withHeaders({ - authorization: "somerandomjwt", - }) - .withBudget({ - // limit up to 200 pages crawled for the entire website - "*": 200, - }) - .withBlacklistUrl([new RegExp("/books").source, "/resume"]) - .build(); +const website = new Website("https://rsseau.fr"); await website.crawl(); console.log(website.getLinks()); diff --git a/npm/android-arm-eabi/package.json b/npm/android-arm-eabi/package.json index 4b82776..08d3bd6 100644 --- a/npm/android-arm-eabi/package.json +++ b/npm/android-arm-eabi/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-android-arm-eabi", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "android" diff --git a/npm/android-arm64/package.json b/npm/android-arm64/package.json index 19e7ee1..21bc1a1 100644 --- a/npm/android-arm64/package.json +++ b/npm/android-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-android-arm64", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "android" diff --git a/npm/darwin-arm64/package.json b/npm/darwin-arm64/package.json index ee8b700..70c8d2d 100644 --- a/npm/darwin-arm64/package.json +++ b/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-darwin-arm64", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "darwin" diff --git a/npm/darwin-universal/package.json b/npm/darwin-universal/package.json index fbd8d39..fd0617e 100644 --- a/npm/darwin-universal/package.json +++ b/npm/darwin-universal/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-darwin-universal", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "darwin" diff --git a/npm/darwin-x64/package.json b/npm/darwin-x64/package.json index b4e59cf..8f75e95 100644 --- a/npm/darwin-x64/package.json +++ b/npm/darwin-x64/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-darwin-x64", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "darwin" diff --git a/npm/freebsd-x64/package.json b/npm/freebsd-x64/package.json index 170f908..1f714af 100644 --- a/npm/freebsd-x64/package.json +++ b/npm/freebsd-x64/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-freebsd-x64", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "freebsd" diff --git a/npm/linux-arm-gnueabihf/package.json b/npm/linux-arm-gnueabihf/package.json index 64f6d23..80661cc 100644 --- a/npm/linux-arm-gnueabihf/package.json +++ b/npm/linux-arm-gnueabihf/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-linux-arm-gnueabihf", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "linux" diff --git a/npm/linux-arm64-gnu/package.json b/npm/linux-arm64-gnu/package.json index aca77fb..83e05ba 100644 --- a/npm/linux-arm64-gnu/package.json +++ b/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-linux-arm64-gnu", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "linux" diff --git a/npm/linux-arm64-musl/package.json b/npm/linux-arm64-musl/package.json index be7b249..137e40d 100644 --- a/npm/linux-arm64-musl/package.json +++ b/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-linux-arm64-musl", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "linux" diff --git a/npm/linux-x64-gnu/package.json b/npm/linux-x64-gnu/package.json index fd64ddc..3135e59 100644 --- a/npm/linux-x64-gnu/package.json +++ b/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-linux-x64-gnu", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "linux" diff --git a/npm/linux-x64-musl/package.json b/npm/linux-x64-musl/package.json index 47c47f2..a4fba8e 100644 --- a/npm/linux-x64-musl/package.json +++ b/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-linux-x64-musl", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "linux" diff --git a/npm/win32-arm64-msvc/package.json b/npm/win32-arm64-msvc/package.json index 4e8af5a..cad1ff2 100644 --- a/npm/win32-arm64-msvc/package.json +++ b/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-win32-arm64-msvc", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "win32" diff --git a/npm/win32-ia32-msvc/package.json b/npm/win32-ia32-msvc/package.json index 7790e5f..e6d0eab 100644 --- a/npm/win32-ia32-msvc/package.json +++ b/npm/win32-ia32-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-win32-ia32-msvc", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "win32" diff --git a/npm/win32-x64-msvc/package.json b/npm/win32-x64-msvc/package.json index 80a96eb..2e0d0b9 100644 --- a/npm/win32-x64-msvc/package.json +++ b/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs-win32-x64-msvc", - "version": "0.0.19", + "version": "0.0.20", "repository": "https://github.com/spider-rs/spider-nodejs", "os": [ "win32" diff --git a/package.json b/package.json index 5ad48b6..faaf157 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@spider-rs/spider-rs", - "version": "0.0.19", + "version": "0.0.20", "main": "index.js", "types": "index.d.ts", "napi": {