Ad

Do Command Only After Getting All The Files

All scripts on my server are on PHP, but one task is possible to resolve only with Node.js.

Node.js gets list of URLs from database, gets content of each page and writes it to files. After that Node.js calls PHP script, which works with saved files.

const filesystem = require('fs');
const requestPromise = require('request-promise');
const mysql = require('mysql');
const database = mysql.createConnection({});

database.connect(function(err){
    if (err){throw err}
    database.query('SELECT * FROM pages', function(err, result) {
        if (err){throw err}
        Object.keys(result).forEach(function(key){
            let page = result[key];
            let url = "https://www.website.com/" + page.url
            requestPromise({})
                .then((html) => {
                    filesystem.writeFile(page.id + '.html', html, function(err){
                        if (err){return console.log(err)}
                    });
                })
                .catch((err) => {console.log(err)})
        })//foreach
    })
})//database.connect

//only after all files have been got and filled!
requestPromise({uri: 'http://127.0.0.1/parse_with_php'})

But the problem is when calling local scripts, not all of files have been saved.

P.S. It takes most of time to get and save content behind the URLs, but not to take few rows from database.

Ad

Answer

Two remarks: querying database requires time too; and I use one more module util. Working code:

const util = require('util');
const mysql = require('mysql');
const filesystem = require('fs');
const requestPromise = require('request-promise');

const database = mysql.createConnection({/*options*/});
const query = util.promisify(database.query).bind(database);

(async () => {
    try {
        const pages = await query('SELECT * FROM pages');
//console.log(pages);
        for (let key of Object.keys(pages)){
            const page = pages[key]
            const url = "https://www.website.com/" + page.url
            const options = {
                uri: url,
                transform: function(body) {return body},
            }
            const html = await requestPromise(options)
            const writeResponse = await filesystem.writeFile(page.id + '.html', html, function(err){if (err){console.log(err)}})
        }
    } finally {
        database.end(); 
//console.log('all files have been saved')
        requestPromise({uri: 'http://127.0.0.1/parse_with_php'})
        process.exit()
    }
})()
Ad
source: stackoverflow.com
Ad