网络数据扒取，http ftp 分类扒取

package spiderFTP; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import org.apache.commons.net.ftp.FTP; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPFile; import org.apache.commons.net.ftp.FTPReply; public class FtpDownload { private FTPClient ftpClient = new FTPClient(); public boolean connect(String ftp, String localDir){ String[] str = ftp.replace("ftp://", "").split("/"); String hostname = str[0]; int port = 21; String username = "anonymous"; String password = "12@download"; localDir = localDir.endsWith("/")? localDir : localDir+"/"; String remote ="/"; for(int i = 1; i < str.length; i++){ remote += str[i] + "/"; } try { connect(hostname, port, username, password); download(remote, localDir); disconnect(); } catch (IOException e) { System.out.println("连接FTP出错："+e.getMessage()); e.printStackTrace(); } return false; } /** * 连接到FTP服务器 * @param hostname 主机名 * @param port 端口 * @param username 用户名 * @param password 密码 * @return 是否连接成功 * @throws IOException */ public boolean connect(String hostname,int port,String username,String password) throws IOException{ System.out.println(hostname +" , "+port+" , "+username+" , "+password); ftpClient.connect(hostname, port); ftpClient.setControlEncoding("GBK"); if(FTPReply.isPositiveCompletion(ftpClient.getReplyCode())){ if(ftpClient.login(username, password)){ return true; } } disconnect(); return false; } /** * 断开与远程服务器的连接 * @throws IOException */ public void disconnect() throws IOException{ if(ftpClient.isConnected()){ ftpClient.disconnect(); } } /** * 从FTP服务器上下载文件,支持断点续传，上传百分比汇报 * @param remote 远程文件路径 * @param local 本地文件路径 * @return 上传的状态 * @throws IOException */ public void download(String remote,String local) throws IOException{ System.out.println("remote,local:"+remote+ " , "+local); //设置被动模式 ftpClient.enterLocalPassiveMode(); //设置以二进制方式传输 ftpClient.setFileType(FTP.BINARY_FILE_TYPE); DownloadStatus result; //检查远程文件是否存在 FTPFile[] files = ftpClient.listFiles(new String(remote.getBytes("GBK"),"iso-8859-1")); System.out.println("共有文件： "+files.length +" 个"); // 检查并创建本地目录 for(int i = 0; i < files.length; i++){ result = downloadFile(files[i], remote, local); // if() } // long lRemoteSize = files[0].getSize(); // System.out.println(files[0].getName()); //本地存在文件，进行断点下载 } private DownloadStatus downloadFile(FTPFile remoteFile,String remoteDir, String local) throws IOException{ DownloadStatus result; File f = new File(local + remoteFile.getName()); if(!f.getParentFile().exists() || !f.getParentFile().isDirectory()){ f.getParentFile().mkdirs(); } long lRemoteSize = remoteFile.getSize(); System.out.println("=====下载======"+remoteFile.getName()+"======="); if(f.exists()){ long localSize = f.length(); //判断本地文件大小是否大于远程文件大小 if(localSize >= lRemoteSize){ System.out.println("本地文件大于远程文件，下载中止"); return DownloadStatus.Local_Bigger_Remote; } //进行断点续传，并记录状态 FileOutputStream out = new FileOutputStream(f,true); ftpClient.setRestartOffset(localSize); InputStream in = ftpClient.retrieveFileStream(new String((remoteDir+remoteFile.getName()).getBytes("GBK"),"iso-8859-1")); byte[] bytes = new byte[1024]; long step = lRemoteSize /100; long process=localSize /step; int c; while((c = in.read(bytes))!= -1){ out.write(bytes,0,c); localSize+=c; long nowProcess = localSize /step; if(nowProcess > process){ process = nowProcess; if(process % 10 == 0) System.out.println("下载进度："+process+"%"); //TODO 更新文件下载进度,值存放在process变量中 } } in.close(); out.close(); boolean isDo = ftpClient.completePendingCommand(); if(isDo){ result = DownloadStatus.Download_From_Break_Success; }else { result = DownloadStatus.Download_From_Break_Failed; } }else { f.createNewFile(); OutputStream out = new FileOutputStream(f); InputStream in= ftpClient.retrieveFileStream(new String((remoteDir+remoteFile.getName()).getBytes("GBK"),"iso-8859-1")); byte[] bytes = new byte[1024]; long step = lRemoteSize /100; long process=0; long localSize = 0L; int c; while((c = in.read(bytes))!= -1){ out.write(bytes, 0, c); localSize+=c; long nowProcess = localSize /step; if(nowProcess > process){ process = nowProcess; if(process % 10 == 0) System.out.println("下载进度："+process+"%"); //TODO 更新文件下载进度,值存放在process变量中 } } in.close(); out.close(); boolean upNewStatus = ftpClient.completePendingCommand(); if(upNewStatus){ result = DownloadStatus.Download_New_Success; }else { result = DownloadStatus.Download_New_Failed; } } return result; } /** * @param args */ public static void main(String[] args) { // TODO Auto-generated method stub FtpDownload myFtp = new FtpDownload(); myFtp.connect("ftp://10.24.189.70/11/AA/","e:/temp/delete2"); } }