2025届必备的降重复率方案实测分析
2026/4/3 21:50:24
作为内蒙古金融行业某上市公司项目负责人,针对集团提出的大文件传输系统需求,本人经过详细调研与技术评估,现提供一套完整的解决方案,确保满足客户对超大文件传输、断点续传、数据安全、国产化兼容等核心要求。
vue-uploader(兼容Vue2/Vue3)+spark-md5(文件哈希计算)localStorage+Redis双缓存(兼容IE8)webkitdirectory属性),生成层级元数据import SparkMD5 from 'spark-md5'; import { uploadFile, resumeUpload } from '@/api/fileUpload'; export default { data() { return { fileList: [], uploading: false, progress: 0, status: '', chunkSize: 1024 * 1024 * 1024, // 1G/片 redisClient: null // 连接Redis(用于断点续传) }; }, mounted() { // 初始化Redis连接(生产环境通过配置中心获取) this.redisClient = new Redis({ host: 'your-redis-host', port: 6379, password: 'your-redis-password' }); }, methods: { async selectFile() { const input = document.createElement('input'); input.type = 'file'; input.webkitdirectory = true; // 支持文件夹上传(Chrome/Firefox) input.multiple = true; input.addEventListener('change', async (e) => { const files = Array.from(e.target.files); this.fileList = files.map(file => ({ name: file.name, size: file.size, status: 'pending', hash: await this.calculateFileHash(file) })); this.startUpload(); }); input.click(); }, // 计算文件哈希(用于断点续传校验) async calculateFileHash(file) { return new Promise((resolve) => { const chunks = Math.ceil(file.size / this.chunkSize); const spark = new SparkMD5.ArrayBuffer(); const reader = new FileReader(); let currentChunk = 0; reader.onload = (e) => { spark.append(e.target.result); currentChunk++; if (currentChunk < chunks) { loadNext(); } else { resolve(spark.end()); } }; const loadNext = () => { const start = currentChunk * this.chunkSize; const end = Math.min(start + this.chunkSize, file.size); reader.readAsArrayBuffer(file.slice(start, end)); }; loadNext(); }); }, // 开始上传(支持断点续传) async startUpload() { this.uploading = true; this.status = ''; for (const file of this.fileList) { const hash = file.hash; const res = await this.checkResumeStatus(hash); // 查询Redis进度 if (res.progress > 0) { this.fileList = this.fileList.map(f => f.hash === hash ? {...f, status: 'resuming'} : f ); await this.resumeUpload(file, res.progress); } else { await this.uploadFile(file); } } this.uploading = false; this.status = 'success'; }, // 检查断点续传状态 async checkResumeStatus(hash) { const progress = await this.redisClient.get(`upload:${hash}:progress`); return { progress: progress ? parseInt(progress) : 0 }; }, // 分片上传(支持断点) async uploadFile(file) { const totalChunks = Math.ceil(file.size / this.chunkSize); const hash = file.hash; for (let i = 0; i < totalChunks; i++) { const start = i * this.chunkSize; const end = Math.min(start + this.chunkSize, file.size); const chunk = file.slice(start, end); const formData = new FormData(); formData.append('file', chunk); formData.append('hash', hash); formData.append('chunk', i); formData.append('total', totalChunks); try { await uploadFile(formData); // 调用后端分片上传接口 const progress = Math.round(((i + 1) / totalChunks) * 100); this.fileList = this.fileList.map(f => f.hash === hash ? {...f, status: 'uploading', progress} : f ); await this.redisClient.set(`upload:${hash}:progress`, progress); // 更新Redis进度 } catch (err) { this.fileList = this.fileList.map(f => f.hash === hash ? {...f, status: 'failed'} : f ); throw new Error(`上传失败:${err.message}`); } } // 合并分片(后端自动触发) await this.mergeChunks(hash, totalChunks); this.fileList = this.fileList.map(f => f.hash === hash ? {...f, status: 'success'} : f ); await this.redisClient.del(`upload:${hash}:progress`); // 清除进度缓存 }, // 合并分片(前端触发后端合并) async mergeChunks(hash, totalChunks) { await fetch('/api/file/merge', { method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({ hash, totalChunks }) }); }, // 格式化文件大小 formatSize(size) { if (size >= 1024 * 1024 * 1024) { return `${(size / (1024 * 1024 * 1024)).toFixed(2)}GB`; } else if (size >= 1024 * 1024) { return `${(size / (1024 * 1024)).toFixed(2)}MB`; } return `${(size / 1024).toFixed(2)}KB`; } } };webkitdirectory属性获取文件夹结构,后端递归创建目录(支持Linux/Windows路径)XMLHttpRequest替代fetch,降级处理分片上传(单文件≤2G)localStorage+Redis双缓存记录进度,浏览器重启后可恢复@RestController@RequestMapping("/api/file")publicclassFileUploadController{@AutowiredprivateFileStorageServicestorageService;@AutowiredprivateRedisTemplateredisTemplate;// 分片上传@PostMapping("/upload")publicResponseEntityuploadChunk(@RequestParam("file")MultipartFilechunk,@RequestParam("hash")Stringhash,@RequestParam("chunk")IntegerchunkNumber,@RequestParam("total")IntegertotalChunks)throwsIOException{// 校验分片哈希StringchunkHash=DigestUtils.md5Hex(chunk.getBytes());if(!chunkHash.equals(hash)){returnResponseEntity.badRequest().body("分片哈希校验失败");}// 存储分片到临时目录(格式:{hash}/{chunkNumber})StringtempDir="temp/"+hash;PathtempPath=Paths.get(tempDir,String.valueOf(chunkNumber));Files.createDirectories(tempPath.getParent());chunk.transferTo(tempPath.toFile());// 更新Redis进度redisTemplate.opsForValue().increment("upload:"+hash+":progress",1);returnResponseEntity.ok().build();}// 合并分片@PostMapping("/merge")publicResponseEntitymergeChunks(@RequestBodyMergeRequestrequest)throwsIOException{Stringhash=request.getHash();IntegertotalChunks=request.getTotalChunks();// 检查所有分片是否上传完成for(inti=0;i<totalChunks;i++){PathchunkPath=Paths.get("temp/"+hash,String.valueOf(i));if(!Files.exists(chunkPath)){returnResponseEntity.badRequest().body("分片缺失:"+i);}}// 创建目标文件(保留层级结构)StringfilePath=storageService.generateFilePath(hash);// 根据业务逻辑生成路径Files.createDirectories(Paths.get(filePath).getParent());// 合并分片(使用NIO提高性能)try(RandomAccessFileraf=newRandomAccessFile(filePath,"rw")){for(inti=0;i<totalChunks;i++){byte[]data=Files.readAllBytes(Paths.get("temp/"+hash,String.valueOf(i)));raf.write(data);Files.delete(Paths.get("temp/"+hash,String.valueOf(i)));// 删除临时分片}}// 记录元数据(文件名、大小、哈希、存储路径)FileInfofileInfo=newFileInfo();fileInfo.setHash(hash);fileInfo.setPath(filePath);fileInfo.setSize(Files.size(Paths.get(filePath)));fileInfo.setCreateTime(LocalDateTime.now());metadataRepository.save(fileInfo);// 清除Redis进度redisTemplate.delete("upload:"+hash+":progress");returnResponseEntity.ok().build();}}// 合并请求DTO@DataclassMergeRequest{privateStringhash;privateIntegertotalChunks;}@ServicepublicclassEncryptionService{// SM4加密(国密)publicbyte[]sm4Encrypt(byte[]data,Stringkey)throwsException{SM4sm4=newSM4();sm4.setKey(key.getBytes(StandardCharsets.UTF_8),SM4.ENCRYPT_MODE);returnsm4.doFinal(data);}// AES加密(国际标准)publicbyte[]aesEncrypt(byte[]data,Stringkey)throwsException{Ciphercipher=Cipher.getInstance("AES/GCM/NoPadding");SecretKeySpeckeySpec=newSecretKeySpec(key.getBytes(StandardCharsets.UTF_8),"AES");GCMParameterSpecgcmParameterSpec=newGCMParameterSpec(128,newbyte[12]);cipher.init(Cipher.ENCRYPT_MODE,keySpec,gcmParameterSpec);byte[]iv=cipher.getIV();byte[]encrypted=cipher.doFinal(data);returnBytes.concat(iv,encrypted);// IV+密文}}// 文件存储服务(支持OBS/本地)@ServicepublicclassFileStorageService{@Value("${storage.type:obs}")privateStringstorageType;@AutowiredprivateObsClientobsClient;// 华为OBS客户端publicvoiduploadFile(StringlocalPath,StringossPath)throwsIOException{if("obs".equals(storageType)){// 上传到华为OBSobsClient.putObject(newPutObjectArgs().bucket("your-obs-bucket").object(ossPath).filename(localPath));}else{// 本地存储(信创环境)Files.copy(Paths.get(localPath),Paths.get(ossPath));}}}# application.ymlspring:datasource:dynamic:primary:mysql# 默认数据库datasource:mysql:url:jdbc:mysql://${mysql.host}:${mysql.port}/${mysql.db}?useSSL=falseusername:${mysql.user}password:${mysql.password}oracle:url:jdbc:oracle:thin:@${oracle.host}:${oracle.port}:${oracle.sid}username:${oracle.user}password:${oracle.password}dm:# 达梦数据库url:jdbc:dm://${dm.host}:${dm.port}/${dm.db}username:${dm.user}password:${dm.password}kingbase:# 人大金仓url:jdbc:kingbase://${kingbase.host}:${kingbase.port}/${kingbase.db}username:${kingbase.user}password:${kingbase.password}jindun-file-transfer仓库application.yml中的数据库/OSS/Redis连接信息mvn clean package -DskipTests(生成jindun-file-transfer-1.0.0.jar)java -jar jindun-file-transfer-1.0.0.jar --spring.profiles.active=prod金盾大文件传输平台V1.0(登记号:2024SR000000)结语:本方案深度适配金融行业需求,兼顾安全性、稳定性与扩展性,源码授权模式可大幅降低集团研发成本。我们承诺60天内完成集团所有项目的集成验证,7×24小时技术支持保障系统稳定运行。期待与集团携手,共同打造金融级大文件传输标杆!
下载完整示例