-
Notifications
You must be signed in to change notification settings - Fork 0
/
writeToFile.js
46 lines (42 loc) · 1.29 KB
/
writeToFile.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
// This version fetches data from the publicstuff API and write it to a file in SQL format.
const fetch = (...args) => import('node-fetch').then(({ default: fetch }) => fetch(...args));
require("dotenv").config();
const fs = require('fs');
async function go() {
try {
const api_url_base = 'https://api.publicstuff.com/app/requests?limit=500&filters[client_id]=819';
let api_url = api_url_base;
const api_key = process.env.api_key;
let next_page = 0;
var stream = fs.createWriteStream("data.sql", { flags: 'a' });
do {
let res = await fetch(api_url, {
method: 'GET',
headers: {
'Authorization': 'Token token=' + api_key
},
})
data = await res.json();
next_page = data.properties.next_page;
api_url = api_url_base + '&page=' + next_page;
await appendSQLFile(stream,data.entities);
} while (next_page);
} catch (err) {
console.log(err);
} finally {
stream.end();
}
}
go();
async function appendSQLFile(stream,rows) {
let sql = 'INSERT INTO avlapp_requests (id, json_data) VALUES ';
try {
for (const row of rows) {
sql = sql + `
(${row.properties.id},'${JSON.stringify(row).replace(/'/g, "''")}'),`;
}
stream.write(sql.slice(0, -1) + ';\n');
} catch (err) {
throw err;
}
}