chore(types): add whitelist blacklist

javascript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.0.48",
+  "version": "0.0.49",
   "description": "A Javascript SDK for Spider Cloud services",
   "scripts": {
     "test": "node --import tsx  --test __tests__/*test.ts",

javascript/src/config.ts

@@ -87,6 +87,16 @@ export interface SpiderParams {
    */
   budget?: Budget;
 
+  /**
+   * The blacklist routes to ignore. This can be a Regex string pattern.
+   */
+  black_list?: string[];
+
+  /**
+   * The whitelist routes to only crawl. This can be a Regex string pattern and used with black_listing.
+   */
+  white_list?: string[];
+
   /**
    * The locale to be used during the crawl.
    */
@@ -230,6 +240,6 @@ export type SpiderCoreResponse = {
   status?: number;
   // The website url.
   url?: string;
-}
+};
 
-export type ChunkCallbackFunction =  (data: SpiderCoreResponse) => void
+export type ChunkCallbackFunction = (data: SpiderCoreResponse) => void;

python/setup.py

@@ -8,7 +8,7 @@ def read_file(fname):
 
 setup(
     name="spider-client",
-    version="0.0.48",
+    version="0.0.49",
     url="https://github.com/spider-rs/spider-clients/tree/main/python",
     author="Spider",
     author_email="[email protected]",

python/spider/spider.py

@@ -377,7 +377,7 @@ def _prepare_headers(self, content_type: str = "application/json"):
         return {
             "Content-Type": content_type,
             "Authorization": f"Bearer {self.api_key}",
-            "User-Agent": f"Spider-Client/0.0.48",
+            "User-Agent": f"Spider-Client/0.0.49",
         }
 
     def _post_request(self, url: str, data, headers, stream=False):

python/spider/spider_types.py

@@ -35,6 +35,12 @@ class RequestParamsDict(TypedDict, total=False):
     # The budget for various resources.
     budget: Optional[Dict[str, int]]
 
+    # The blacklist routes to ignore. This can be a Regex string pattern.
+    black_list: Optional[List[str]]
+
+    # The whitelist routes to only crawl. This can be a Regex string pattern and used with black_listing.
+    white_list: Optional[List[str]]
+
     # The locale to be used during the crawl.
     locale: Optional[str]
 
@@ -114,4 +120,4 @@ class RequestParamsDict(TypedDict, total=False):
     chunking_alg: Optional[ChunkingAlgDict]
 
 
-JsonCallback = Callable[[dict], None]
+JsonCallback = Callable[[dict], None]