Skip to content

Commit

Permalink
refactor(scrape.js): simplify the code
Browse files Browse the repository at this point in the history
  • Loading branch information
erdie committed Jan 16, 2024
1 parent 85e5a13 commit af6d64f
Showing 1 changed file with 14 additions and 38 deletions.
52 changes: 14 additions & 38 deletions src/scrape.js
Original file line number Diff line number Diff line change
@@ -1,54 +1,30 @@
const fs = require('fs')
const got = require('got')
const jsdom = require("jsdom")
const { JSDOM } = jsdom
const { JSDOM } = require("jsdom")

const url= 'https://www.iana.org/domains/root/db'

got(url).then(response => {
const dom = new JSDOM(response.body)

// select element
const selectDomain = dom.window.document.querySelectorAll('table td:nth-child(1) span a')
const selectType = dom.window.document.querySelectorAll('table tr td:nth-child(2)')
const selectTldManager = dom.window.document.querySelectorAll('table tr td:nth-child(3)')

const domainArray = []
selectDomain.forEach(fistColumn => {
domainArray.push(fistColumn.textContent)
});

const typeArray = []
selectType.forEach(secondColumn => {
typeArray.push(secondColumn.textContent)
});

const managerArray = []
selectTldManager.forEach(thirdColumn => {
managerArray.push(thirdColumn.textContent)
});

// compile all object into json format
const compileData = []
domainArray.forEach((item, index) => {
const row = {
'domain': domainArray[index],
'type': typeArray[index],
'tldManager' : managerArray[index]
}
compileData.push(row)
const compileData = Array.from(selectDomain).map((item, index) => {
return {
'domain': item.textContent,
'type': selectType[index].textContent,
'tldManager' : selectTldManager[index].textContent
}
})

const dataJSON = JSON.stringify(compileData)
// console.log(dataJSON)

fs.writeFile('../public/iana-tld.json', dataJSON, err => {
if (err) {
console.log('Error writing file', err)
} else {
console.log('Successfully wrote file')
}
})
fs.writeFile('./public/iana-tld.json', JSON.stringify(compileData), err => {
if (err) {
console.log('Error writing file', err)
} else {
console.log('Successfully wrote file')
}
})
}).catch(err => {
console.log(err)
})

0 comments on commit af6d64f

Please sign in to comment.