How to follow all links in CasperJS?

casperjs, hyperlink, javascript, phantomjs, web-crawler

Solution

I have this script that first will get all links from a page then save 'href' attributes to an array, then will iterate over this array and then open each link one by one and echo the url :

var casper = require('casper').create({
    logLevel:"verbose",
    debug:true
});
var links;

casper.start('http://localhost:8000');

casper.then(function getLinks(){
     links = this.evaluate(function(){
        var links = document.getElementsByTagName('a');
        links = Array.prototype.map.call(links,function(link){
            return link.getAttribute('href');
        });
        return links;
    });
});
casper.then(function(){
    this.each(links,function(self,link){
        self.thenOpen(link,function(a){
            this.echo(this.getCurrentUrl());
        });
    });
});
casper.run(function(){
    this.exit();
});

Problem

I'm having trouble clicking all JavaScript based links in a DOM and saving the output. The links have the form ``` <a id="html" href="javascript:void(0);" onclick="goToHtml();">HTML</a> ``` the following code works great: ``` var casper = require('casper').create(); var fs = require('fs'); var firstUrl = 'http://www.testurl.com/test.html'; var css_selector = '#jan_html'; casper.start(firstUrl); casper.thenClick(css_selector, function(){ console.log("whoop"); }); casper.waitFor(function check() { return this.getCurrentUrl() != firstUrl; }, function then() { console.log(this.getCurrentUrl()); var file_title = this.getTitle().split(' ').join('_') + '.html'; fs.write(file_title, this.getPageContent()); }); casper.run(); ``` However, how can I get this to work with a selector of "a", clicking all available links and saving content? I'm not sure how to get the clickWhileSelector to remove nodes from the selector as is done here: Click on all links matching a selector

Original source

Related problems