Jak pobierać obrazy z witryny za pomocą phantomjs

Question

Chcę zapisać niektóre obrazy z witryny. W tej chwili mogę uzyskać ścieżki do obrazów, ale nie mam pojęcia, jak zdobyć i zapisać obrazy za pomocą phantomJs.

findRotationTeaserImages = -> 
    paths = page.evaluate -> 
    jQuery('.rotate img').map(-> return this.src).get() 

    for path, i in paths 
    console.log(path); 
    //save the image 

Answer 1

wiem to stare pytanie, ale robisz to po prostu zapisując wymiary i położenie każdego obrazu na obiekcie, a następnie zmieniając stronę phantomjs page.clipRect, aby metoda page.render() renderowała tylko obszar, w którym obraz jest. Oto przykład, zgarniania wiele obrazów z http://dribbble.com/:

var page = require('webpage').create(); 

page.open('http://dribbble.com/', function() { 

    page.includeJs('//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js',function() { 

     var images = page.evaluate(function() { 
      var images = []; 
      function getImgDimensions($i) { 
       return { 
        top : $i.offset().top, 
        left : $i.offset().left, 
        width : $i.width(), 
        height : $i.height() 
       } 
      } 
      $('.dribbble-img img').each(function() { 
       var img = getImgDimensions($(this)); 
       images.push(img); 
      }); 

      return images; 
     }); 

     images.forEach(function(imageObj, index, array){ 
      page.clipRect = imageObj; 
      page.render('images/'+index+'.png') 
     }); 

     phantom.exit(); 
    }); 
}); 

Answer 2

rozwiązać ten problem poprzez rozpoczęcie procesu potomnego uruchomienie skryptu węzła, który pobrać obrazy:

phantomJs Scenariusz:

findRotationTeaserImages = -> 
    paths = page.evaluate -> 
    jQuery('.rotate img').map(-> return this.src).get() 

    args = ('loadRotationTeaser.js ' + paths.join(' ')).split(' ') 

    child_process.execFile("node", args, null, (err, stdout, stderr) -> 
    phantom.exit() 
) 

nodeJs skrypt

http = require('http-get'); 

args = process.argv.splice(2) 

for path, i in args 
    http.get path, 'public/images/rotationTeaser/img' + i + '.jpeg', (error, result) -> 

Answer 3

Jest teraz innym sposobem, aby to zrobić.

var fs = require("fs"); 
var imageBase64 = page.evaluate(function(){ 
    var canvas = document.createElement("canvas"); 
    canvas.width =img.width; 
    canvas.height =img.height; 
    var ctx = canvas.getContext("2d"); 
    ctx.drawImage(img, 0, 0);  
    return canvas.toDataURL ("image/png").split(",")[1]; 
}) 
fs.write("file.png",atob(imageBase64),'wb'); 

Answer 4

W przypadku wymiarów obrazu są znane:

 


    var webPage = require('webpage'); 

    /** 
    * Download image with known dimension. 
    * @param src Image source 
    * @param dest Destination full path 
    * @param width Image width 
    * @param height Image height 
    * @param timeout Operation timeout 
    * @param cbk Callback (optional) 
    * @param cbkParam Parameter to pass back to the callback (optional) 
    */ 
    function downloadImg(src, dest, width, height, timeout, cbk, cbkParam) { 
     var page = webPage.create(); 

     page.settings.resourceTimeout = timeout; //resources loading timeout(ms) 
     page.settings.webSecurityEnabled = false; //Disable web security 
     page.settings.XSSAuditingEnabled = false; //Disable web security 

     page.open(src, function(status) { 

      // missing images sometime receive text from server 
      var success = status == 'success' && !page.plainText; 

      if (success) { 
       page.clipRect = { 
        top: 0, 
        left: 0, 
        width: width, 
        height: height 
       }; 
       page.render(dest); 

      } 

      cbk && cbk(success, cbkParam); 
      page.close(); 
     }); 
    }; 

 

Answer 5

jakie miałem naprawdę dużo kłopotów przy użyciu metody render. Na szczęście w końcu mam dwa lepsze rozwiązania. Oto kod, którego użyłem w moim projekcie. Pierwsze rozwiązanie ma problem z aktualizacją pliku cookie, więc nie działa dobrze podczas pobierania obrazu captcha. Obie metody spowodują nowe żądanie http. Ale z kilkoma modyfikacjami, drugi może anulować takie żądanie.

Pierwsza pobiera plik cookie z phantomJs i tworzy nowe żądanie http za pomocą request. Drugi używa base64 do przekazania obrazu.

async download(download_url, stream) { 
    logger.profile(`download(download_url='${download_url}')`); 
    let orig_url = await this.page.property('url'); 
    download_url = url.resolve(orig_url, download_url); 
    let cookies = await this.page.property('cookies'); 
    let jar = request.jar(); 
    for (let cookie of cookies) { 
     if (cookie.name !== undefined) { 
      cookie.key = cookie.name; 
      delete cookie.name; 
     } 
     if (cookie.httponly !== undefined) { 
      cookie.httpOnly = cookie.httponly; 
      delete cookie.httponly; 
     } 
     if (cookie.expires !== undefined) 
      cookie.expires = new Date(cookie.expires); 
     jar.setCookie(new Cookie(cookie), download_url, {ignoreError: true}); 
    } 
    let req = request({ 
     url: download_url, 
     jar: jar, 
     headers: { 
      'User-Agent': this.user_agent, 
      'Referer': orig_url 
     } 
    }); 
    await new Promise((resolve, reject) => { 
     req.pipe(stream) 
      .on('close', resolve) 
      .on('error', reject); 
    }); 
    // Due to this issue https://github.com/ariya/phantomjs/issues/13409, we cannot set cookies back 
    // to browser. It is said to be redesigned, but till now (Mar 31 2017), no change has been made. 
    /*await Promise.all([ 
     new Promise((resolve, reject) => { 
      req.on('response',() => { 
       jar._jar.store.getAllCookies((err, cookies) => { 
        if (err) { 
         reject(err); 
         return; 
        } 
        cookies = cookies.map(x => x.toJSON()); 
        for (let cookie of cookies) { 
         if (cookie.key !== undefined) { 
          cookie.name = cookie.key; 
          delete cookie.key; 
         } 
         if (cookie.httpOnly !== undefined) { 
          cookie.httponly = cookie.httpOnly; 
          delete cookie.httpOnly; 
         } 
         if (cookie.expires instanceof Date) { 
          cookie.expires = cookie.expires.toGMTString(); 
          cookie.expiry = cookie.expires.toTime(); 
         } 
         else if (cookie.expires == Infinity) 
          delete cookie.expires; 
         delete cookie.lastAccessed; 
         delete cookie.creation; 
         delete cookie.hostOnly; 
        } 
        this.page.property('cookies', cookies).then(resolve).catch(reject); 
       }); 
      }).on('error', reject); 
     }), 
     new Promise((resolve, reject) => { 
      req.pipe(fs.createWriteStream(save_path)) 
       .on('close', resolve) 
       .on('error', reject); 
     }) 
    ]);*/ 
    logger.profile(`download(download_url='${download_url}')`); 
} 
async download_image(download_url, stream) { 
    logger.profile(`download_image(download_url='${download_url}')`); 
    await Promise.all([ 
     new Promise((resolve, reject) => { 
      this.client.once('donwload image', data => { 
       if (data.err) 
        reject(err); 
       else 
        stream.write(Buffer.from(data.data, 'base64'), resolve); 

      }); 
     }), 
     this.page.evaluate(function (url) { 
      var img = new Image(), callback = function (err, data) { 
       callPhantom({ 
        event: 'donwload image', 
        data: { 
         err: err && err.message, 
         data: data 
        } 
       }); 
      }; 
      img.onload = function() { 
       var canvas = document.createElement("canvas"); 
       canvas.width = img.width; 
       canvas.height = img.height; 
       canvas.getContext("2d").drawImage(img, 0, 0); 
       callback(null, canvas.toDataURL("image/png").replace(/^data:image\/(png|jpg);base64,/, "")); 
      }; 
      img.onerror = function() { 
       callback(new Error('Failed to fetch image.')); 
      }; 
      img.src = url; 
     }, download_url) 
    ]); 
    logger.profile(`download_image(download_url='${download_url}')`); 
}