Writing web scraped data to JSON file using NodeJS

Question

I'm writing an application that scrapes fan sites for characters as a practice exercise. Currently I have an array of URLs that I am looping through and scraping the data I want, then outputting this data to a output.json file to store for later. I am having issues with my formatting when writing to this file.

Maybe I should store my data differently, I am open to suggestions on best practices/other methods. I would just like this data accessible later.

server.js

var express = require('express');
var cheerio = require('cheerio');
var app = express();
var rp = require('request-promise');
var fsp = require('fs-promise');

app.get('/', function(req, res){

  urls = [
    'fansite.com/boss1', 'fansite.com/boss2'
  ];


  function parse(html) {
    var bosses = require('./output.json');
    var $ = cheerio.load(html);
    $('.page-header__title').filter(function () {
      var data = $(this);
      name = data.text();
      bosses.name = name;
    })
    return bosses;
  }
  var append = file => content => fsp.appendFile(file, JSON.stringify(content, null, 2));

  urls.forEach(function (url) {
    rp(url)
    .then(parse)
    .then(append('output.json'))
    .then(() => console.log('Success'))
    .then(res.send('Bosses Updated.'))  
    .catch(err => console.log('Error:', err));
  });
})

app.listen('8081')
console.log('Running on port 8081');
exports = module.exports = app;

output.json

{

}{
  "name": "Boss1"
}{
  "name": "Boss2"
}

Paul · Accepted Answer · 2018-01-29 19:33:32Z

1

You're better off just modifying the in-memory javascript object, and then saving it all to the file in an overwrite / replace kind of approach, rather than appending to the file (unless you expect the file to become so huge that it breaks memory limits).

To do that, just maintain an in-memory copy of the data and then just write it out: fs.writeFile(fileName, JSON.stringify(content, null, 4));

Otherwise, you have to figure out how to insert the new object inside the old one, or risk making it invalid json.

answered Jan 29, 2018 at 19:33

Paul

36.4k12 gold badges98 silver badges124 bronze badges

Sign up to request clarification or add additional context in comments.

Collectives™ on Stack Overflow

Writing web scraped data to JSON file using NodeJS

1 Answer 1

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Comments

Your Answer

Sign up or log in

Post as a guest

Related