Grabbing json data of county map

Find a lot of resources on the Internet, either the data is too old, data is wrong, or to charge, so I grabbed Ali's data.

data source

Project source code

Data is asynchronously fetched and written, so axios is used. The node version is 12.10.0.
Introduce needed resources:

import http from 'axios'
import fs from 'fs'

Address and address splicing function:

const baseUrl = 'http://datavmap-public.oss-cn-hangzhou.aliyuncs.com/areas/'

function ug(sort, code1, code2 = '00') {
    return baseUrl + sort + '/51' + code1 + code2 + '.json'
}

In nodejs, asynchronous interfaces are callback functions, which are not easy to use here, and are converted to promise-based:

function promisify(fn, t, argsNum = 3) { //T: this args: Number of parameters
    return (...args) => {
        let arr = Array.from(args).slice(0, argsNum - 2)
        return new Promise((resolve, reject) => {
            fn.call(t, ...arr, args[argsNum - 2] || {}, (err, res) => {
                return err ? reject(err) : resolve(res)
            })
        })
    };
}

const rmdir = promisify(fs.rmdir, fs),
    mkdir = promisify(fs.mkdir, fs),
    appendFile = promisify(fs.appendFile, fs, 4)

Promises are easier to use, code readability is high, and the corresponding performance will decline a little. The bluebird performance has been optimized, which is higher than promise. If the data is relatively small, do it yourself first.

There are two ways of acquisition, concurrent acquisition:

const children = [...Array(35).keys()]
const bound = [...Array(100).keys()]
bound.shift()
async function getConcurrent() {
    try {
        await rmdir('./map', {recursive: true})
        await mkdir('./map', {recursive: true})
    } catch (e) {
        console.log(e)
    }
    children.map(async (i) => {
        let code1 = i.toString().padStart(2, '0')
        try {
            let r = await http.get(ug('children', code1))
            await (async (r) => {
                await console.log('Obtain children/51' + code1 + '00 Success')
                await appendFile('./map/51' + code1 + '00.json', JSON.stringify(r.data))
                console.log('Preservation children/51' + code1 + '00 Success')
            })(r)
        } catch (e) {
        }
        bound.map(async (j) => {
            let code2 = j.toString().padStart(2, '0')
            try {
                let r = await http.get(ug('bound', code1, code2))
                await (async (r) => {
                    await console.log('###Get bound/51'+ code1 + code2 +'success')
                    await appendFile('./map/51' + code1 + code2 + '.json', JSON.stringify(r.data))
                    console.log('###Save bound/51'+ code1 + code2 + success')
                })(r)
            } catch (e) {
            }
        })
    })
}

Secondary:

async function getSuccessive() {
    try {
        await rmdir('./map', {recursive: true})
        await mkdir('./map', {recursive: true})
    } catch (e) {
        console.log(e)
    }

    for (let i = 0; i < 35; i++) {
        let code1 = i.toString().padStart(2, '0')
        try {
            let r = await http.get(ug('children', code1))
            await (async (r) => {
                await console.log('Obtain children/51' + code1 + '00 Success')
                await appendFile('./map/51' + code1 + '00.json', JSON.stringify(r.data))
                console.log('Preservation children/51' + code1 + '00 Success')
            })(r)
        } catch (e) {
        }

        for (let j = 1; j < 100; j++) {
            let code2 = j.toString().padStart(2, '0')
            try {
                let r = await http.get(ug('bound', code1, code2))
                await (async (r) => {
                    await console.log('###Get bound/51'+ code1 + code2 +'success')
                    await appendFile('./map/51' + code1 + code2 + '.json', JSON.stringify(r.data))
                    console.log('###Save bound/51'+ code1 + code2 + success')
                })(r)
            } catch (e) {
            }
        }
    }
}

Complications are significantly faster than secondary ones.
Function:

function run(option = false) {
    option ? getSuccessive() : getConcurrent()
}

run()

Grabbing the modified data and numbers of other areas will do, and the data of National Geographic Information Network will be grabbed in the future.

Tags: Javascript JSON axios network

Posted on Fri, 11 Oct 2019 09:15:05 -0700 by phplearner