Ftp Resume Transfer
2024-06-25
5 min read
Resumable transfer significantly improves the efficiency of file transfers,It is very efficient and convenient.Let’s try it in java
Dependencies
Using commons-net 3.11.1 under spring-boot-starter-parent 3.2.6,otherwise fatal error might been occur.
10 <parent>
11 <groupId>org.springframework.boot</groupId>
12 <artifactId>spring-boot-starter-parent</artifactId>
13 <version>3.2.6</version>
14 <relativePath/> <!-- lookup parent from repository -->
15 </parent>
16
17 <dependency>
18 <groupId>commons-net</groupId>
19 <artifactId>commons-net</artifactId>
20 <version>3.11.1</version>
21 </dependency>
FTP Client
You will see the class org.apache.commons.net.ftp.FTPClient from commons-net.
10 package com.neoemacs.video.upload.tool.server.service;
11
12 import java.io.IOException;
13 import org.apache.commons.net.ftp.FTPClient;
14 import org.springframework.beans.factory.annotation.Value;
15 import org.springframework.context.annotation.Bean;
16 import org.springframework.context.annotation.Configuration;
17
18 @Configuration
19 public class FtpClientConfiguration {
20
21 @Value("${ftp.server}")
22 private String ftpServer;
23
24 @Value("${ftp.account}")
25 private String ftpAccount;
26
27 @Value("${ftp.password}")
28 private String ftpPassword;
29
30 @Bean
31 public FTPClient ftpClient() {
32 FTPClient ftpClient = new FTPClient();
33 try {
34 ftpClient.connect(ftpServer);
35 ftpClient.login(ftpAccount, ftpPassword);
36 ftpClient.enterLocalPassiveMode();
37 ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE);
38 } catch (IOException e) {
39 throw new RuntimeException("Failed to create FTP client", e);
40 }
41 return ftpClient;
42 }
43 }
Define Interface
Every ftp account have privilege to access a particular director which indeed is your client’s root. Obviously workdir is a sub director under the ftp server’s root. Here use two paramter to make the function’s purpose more clear.
10 /**
11 * Program wil upload file on by one via breanpoint resume.
12 *
13 * e.g. uploadFile("C:/Desktop/tmp", "/record_rtp/test/20240309")
14 * Files located at C:/Desktop/tmp will upload to ${workdir}/record_rtp/test/20240309/
15 *
16 */
17 public void uploadFile(String localFileDir, String ftpDir) throws IOException;
File Offset
The principle of it is query list files info from ftp server,use the size of ftp file as local file’s offset.
10 /**
11 * get file uploaded offset from ftp server
12 */
13 private long checkFileOffsetOnFtp(String fileName) throws IOException {
14 String remoteFile = fileName;
15 FTPFile[] files = ftpClient.listFiles(remoteFile);
16 if (files.length == 1 && files[0].isFile()) {
17 return files[0].getSize();
18 }
19 return 0;
20 }
Start Update
To start update,we need constitute a inputstream and skip it’s offset,and use a remote addr to recive stream.
10
11 @Autowired
12 FTPClient ftpClient;
13
14 File localFile = new File(localFileDir);
15 InputStream inputStream = new FileInputStream(localFile);
16
17 String ftpFilePath = " ";
18 long offset = checkFileOffsetOnFtp(ftpFilePath);
19 inputStream.skip(offset);
20
21 ftpClient.setRestartOffset(offset);
22 boolean done = ftpClient.storeFile(ftpFilePath, inputStream);
Monitor Progress
The CopyStreamListener#bytesTransferred could obtains the progress of uploading.You need override the bytesTransferred , totalBytesTransferred gives the uploaded bytes but do not contains offset bytes from previously process.You could use CopyStreamListener divide by fileSize to got the uploading progress when both of their unit are bytes.
10 ftpClient.setCopyStreamListener(new CopyStreamListener() {
11 private long megsTransferred = 0;
12
13 @Override
14 public void bytesTransferred(long totalBytesTransferred,
15 int bytesTransferred, long streamSize) {
16 long megs = totalBytesTransferred / (1024 * 1024);
17 if (megs > megsTransferred) {
18 megsTransferred = megs;
19 log.info("==== [progress]: {}",
20 (double) (totalBytesTransferred + offset) / fileSize * 100);
21 }
22 }
23
24 @Override
25 public void bytesTransferred(CopyStreamEvent event) {
26 // 此方法在复制流事件时调用
27 }
28 });
Don’t drive bytesTransferred frequency to high.Case It’s might slow down the uploading speed.
Full Code
1 package com.minxiot.video.upload.tool.server.service;
2
3 import java.io.File;
4 import java.io.FileInputStream;
5 import java.io.IOException;
6 import java.io.InputStream;
7 import org.apache.commons.net.ftp.FTPClient;
8 import org.apache.commons.net.ftp.FTPFile;
9 import org.apache.commons.net.io.CopyStreamEvent;
10 import org.apache.commons.net.io.CopyStreamListener;
11 import org.springframework.beans.factory.annotation.Autowired;
12 import org.springframework.stereotype.Service;
13 import lombok.extern.slf4j.Slf4j;
14
15 @Slf4j
16 @Service
17 public class FileUploadService {
18
19 @Autowired
20 FTPClient ftpClient;
21
22 public void uploadFile(String localFileDir, String ftpDir) throws IOException {
23 File localDir = new File(localFileDir);
24 if (!localDir.isDirectory()) {
25 throw new IllegalArgumentException("The provided path is not a directory");
26 }
27
28 for (File eachLocalFile : localDir.listFiles()) {
29 if (eachLocalFile.isFile()) {
30 boolean isConnected = ftpClient.isConnected();
31 log.info("==== [ftp.isConnected]: {} ", isConnected);
32 log.info("==== [local.fileName]: {} ", eachLocalFile.getAbsolutePath());
33 String ftpFileName = ftpDir + eachLocalFile.getName();
34 log.info("==== [ftp.fileName]: {}", ftpFileName);
35 if (!ftpClient.changeWorkingDirectory(ftpDir)) {
36 log.error("==== [ftp.error]: {}, Cannot change to working Dir:{}",
37 ftpClient.getReplyString(), ftpDir);
38 continue;
39 }
40 long offset = checkFileOffsetOnFtp(ftpFileName);
41 log.info("==== [ftp.fileName.offset]: {}:{}", ftpFileName, offset);
42 try (InputStream inputStream = new FileInputStream(eachLocalFile)) {
43 if (offset > 0) {
44 inputStream.skip(offset);
45 ftpClient.setRestartOffset(offset);
46 }
47 long fileSize = eachLocalFile.length();
48 ftpClient.setCopyStreamListener(new CopyStreamListener() {
49 private long megsTransferred = 0;
50
51 @Override
52 public void bytesTransferred(long totalBytesTransferred,
53 int bytesTransferred, long streamSize) {
54 long megs = totalBytesTransferred / (1024 * 1024);
55 if (megs > megsTransferred) {
56 megsTransferred = megs;
57 log.info("==== [progress]: {}",
58 (double) (totalBytesTransferred + offset) / fileSize * 100);
59 }
60 }
61
62 @Override
63 public void bytesTransferred(CopyStreamEvent event) {
64 // 此方法在复制流事件时调用
65 }
66 });
67
68 boolean done = ftpClient.storeFile(ftpFileName, inputStream);
69 if (!done) {
70 log.error("==== [ftp.error]: ReplyCode:{},ReplyString:{}",
71 ftpClient.getReplyCode(), ftpClient.getReplyString());
72 }
73 } catch (IOException e) {
74 log.error("==== [ftp.error]: {}", e);
75 continue;
76 }
77 }
78 }
79 }
80
81 /**
82 * get file uploaded offset from ftp server
83 */
84 private long checkFileOffsetOnFtp(String fileName) throws IOException {
85 String remoteFile = fileName;
86 FTPFile[] files = ftpClient.listFiles(remoteFile);
87 if (files.length == 1 && files[0].isFile()) {
88 return files[0].getSize();
89 }
90 return 0;
91 }
92
93 }