Ad
Do Command Only After Getting All The Files
All scripts on my server are on PHP, but one task is possible to resolve only with Node.js.
Node.js gets list of URLs from database, gets content of each page and writes it to files. After that Node.js calls PHP script, which works with saved files.
const filesystem = require('fs');
const requestPromise = require('request-promise');
const mysql = require('mysql');
const database = mysql.createConnection({});
database.connect(function(err){
if (err){throw err}
database.query('SELECT * FROM pages', function(err, result) {
if (err){throw err}
Object.keys(result).forEach(function(key){
let page = result[key];
let url = "https://www.website.com/" + page.url
requestPromise({})
.then((html) => {
filesystem.writeFile(page.id + '.html', html, function(err){
if (err){return console.log(err)}
});
})
.catch((err) => {console.log(err)})
})//foreach
})
})//database.connect
//only after all files have been got and filled!
requestPromise({uri: 'http://127.0.0.1/parse_with_php'})
But the problem is when calling local scripts, not all of files have been saved.
P.S. It takes most of time to get and save content behind the URLs, but not to take few rows from database.
Ad
Answer
Two remarks: querying database requires time too; and I use one more module util. Working code:
const util = require('util');
const mysql = require('mysql');
const filesystem = require('fs');
const requestPromise = require('request-promise');
const database = mysql.createConnection({/*options*/});
const query = util.promisify(database.query).bind(database);
(async () => {
try {
const pages = await query('SELECT * FROM pages');
//console.log(pages);
for (let key of Object.keys(pages)){
const page = pages[key]
const url = "https://www.website.com/" + page.url
const options = {
uri: url,
transform: function(body) {return body},
}
const html = await requestPromise(options)
const writeResponse = await filesystem.writeFile(page.id + '.html', html, function(err){if (err){console.log(err)}})
}
} finally {
database.end();
//console.log('all files have been saved')
requestPromise({uri: 'http://127.0.0.1/parse_with_php'})
process.exit()
}
})()
Ad
source: stackoverflow.com
Related Questions
- → How to update data attribute on Ajax complete
- → October CMS - Radio Button Ajax Click Twice in a Row Causes Content to disappear
- → Octobercms Component Unique id (Twig & Javascript)
- → Passing a JS var from AJAX response to Twig
- → Laravel {!! Form::open() !!} doesn't work within AngularJS
- → DropzoneJS & Laravel - Output form validation errors
- → Import statement and Babel
- → Uncaught TypeError: Cannot read property '__SECRET_DOM_DO_NOT_USE_OR_YOU_WILL_BE_FIRED' of undefined
- → React-router: Passing props to children
- → ListView.DataSource looping data for React Native
- → Can't test submit handler in React component
- → React + Flux - How to avoid global variable
- → Webpack, React & Babel, not rendering DOM
Ad