docs-scraper.config.json

{
  "index_uid": "production",
  "start_urls": [
      "https://www.meilisearch.com/docs"
  ],
  "sitemap_urls": [
      "https://www.meilisearch.com/sitemap.xml"
  ],
  "selectors": {
      "lvl0": {
          "selector": "#sidebar-list .sidebar-link[data-active=true] ~ .sidebar-category",
          "global": true,
          "default_value": "Documentation"
      },
      "lvl1": {
          "selector": ".docs-main h1",
          "global": true
      },
      "lvl2": ".docs-main h2",
      "lvl3": ".docs-main h3",
      "lvl4": ".docs-main h4",
      "lvl5": ".docs-main h5",
      "text": ".docs-main p, .docs-main li, .docs-main td"
  },
  "strip_chars": " .,;:#",
  "scrap_start_urls": true,
  "custom_settings": {
    "embedders": {
        "default": {
          "source":  "openAi",
          "apiKey": "embedderApiKey",
          "model": "text-embedding-ada-002",
          "documentTemplate": "{% if doc.hierarchy_lvl0 != null %}Starting with the theme '{{doc.hierarchy_lvl0}}'. {% endif %}{% if doc.hierarchy_lvl1 != null %}A document titled '{{doc.hierarchy_lvl1}}'. {% endif %}{% if doc.hierarchy_lvl2 != null %}Under the section '{{doc.hierarchy_lvl2}}'. {% endif %}{% if doc.hierarchy_lvl3 != null %}This is further divided into '{{doc.hierarchy_lvl3}}'. {% endif %}{% if doc.hierarchy_lvl4 != null %}Which includes '{{doc.hierarchy_lvl4}}'. {% endif %}{% if doc.hierarchy_lvl5 != null %}Specifically focusing on '{{doc.hierarchy_lvl5}}'. {% endif %}{% if doc.anchor != null %}It's about {{doc.anchor}}. {% endif %}{% if doc.content != null %}It discusses {{doc.content|truncatewords: 200}}{% endif %}"
        }
      },
      "searchableAttributes": [
          "hierarchy_lvl1",
          "hierarchy_lvl2",
          "hierarchy_lvl3",
          "hierarchy_lvl4",
          "hierarchy_lvl5",
          "content",
          "hierarchy_lvl0"
      ],
      "synonyms": {
          "large language model": [
            "llm"
          ],
          "llm": [
            "large language model"
          ],
          "relevancy": [
              "relevant",
              "relevance"
          ],
          "relevant": [
              "relevancy",
              "relevance"
          ],
          "relevance": [
              "relevancy",
              "relevant"
          ]
      }
  },
  "min_indexed_level": 2,
  "selectors_exclude": [
      "#page-navigation",
      "#feedback"
  ]
}