fix bug:使用旧接口请求合并文件时报错

dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/controller/CompatibleOldFileController.java

@@ -323,7 +323,7 @@ public class CompatibleOldFileController {
         }
         // 1.先上传分片信息
         IFileStorageService service = FileStorageFactory.getService();
-        service.upload(service.getChuckBucketName(null),md5 + "/" + chunkNumber + ".chunk",file.getInputStream());
+        service.uploadChuck(service.getChuckBucketName(null),md5,file.getInputStream(),totalChunks,chunkNumber);
         // 2.更新fileInfo
         if (StrUtil.isBlank(fileInfo.getFileMd5())){
             fileInfo.setFileMd5(md5);
@@ -351,7 +351,7 @@ public class CompatibleOldFileController {
     @SneakyThrows
     @RequestMapping(value = "/common/merge_multipart")
     @ResponseBody
-    public String multipartUpload() {
+    public String mergeMultipart() {
         if (StrUtil.isBlank(OldFileAppContext.getContext().getUploadId())){
             return INCORRECT_PARAMETERS;
         }
@@ -372,13 +372,18 @@ public class CompatibleOldFileController {
             return INCORRECT_PARAMETERS;
         }
         fileMd5Service.ensureFilePath(fileMd5,fileInfo.getFileName());
-        service.composeObject(fileInfo.getCreator(),chunks,fileMd5.getFilePath());
-        // 更新Md5值
-        fileMd5.setUploadStatus(FileCommonConst.UPLOAD_SUCCESS);
-        fileMd5Service.updateById(fileMd5);
+        boolean composeObject = service.composeObject(fileInfo.getCreator(), chunks, fileMd5.getFilePath());
         JSONObject jsonObj = new JSONObject();
-        jsonObj.put("Result", "success");
-        jsonObj.put("ResultMsg", "");
+        if (composeObject){
+            // 更新Md5值
+            fileMd5.setUploadStatus(FileCommonConst.UPLOAD_SUCCESS);
+            fileMd5Service.updateById(fileMd5);
+            jsonObj.put("Result", "success");
+            jsonObj.put("ResultMsg", "");
+        }else {
+            jsonObj.put("Result", "failure");
+            jsonObj.put("ResultMsg", "合并失败");
+        }
         return jsonObj.toJSONString();
     }
     /***

dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/handler/OldFileContextHandler.java

@@ -24,6 +24,7 @@ public class OldFileContextHandler implements HandlerInterceptor {
         context.setWorkOrderId(request.getParameter(OldFileContext.WORK_ORDER_ID));
         context.setProjectId(request.getParameter(OldFileContext.PROJECT_ID));
         context.setOverwrite(BooleanUtil.toBoolean(request.getParameter(OldFileContext.OVERWRITE)));
+        context.setUploadId(request.getParameter(OldFileContext.UPLOAD_ID));
         AppContext.getContext().setAccountId(request.getParameter(OldFileContext.SYSTEM_ID));
         return true;
     }

dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/IFileStorageService.java

@@ -209,4 +209,17 @@ public interface IFileStorageService {
      * Update By lijie 2021/12/9 14:33
      */
     void batchDelete(Map<String, Set<String>> deleteFileMap);
+    /***
+     * Description: 上传分片
+     * @param chuckBucketName : bucketName
+     * @param fileMd5 : 文件的md5值
+     * @param inputStream : 文件流
+     * @param totalChunks : 总分片数
+     * @param currentChuckNumber : 当前分片数
+     * return : void
+     * @author : lijie
+     * Update By 2021/12/17 15:18
+     */
+    void uploadChuck(String chuckBucketName, String fileMd5, InputStream inputStream,Integer totalChunks,
+                     Integer currentChuckNumber);
 }

dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/impl/HdfsStorageServiceImpl.java

@@ -56,6 +56,22 @@ public class HdfsStorageServiceImpl implements IFileStorageService {
             FileExceptionHandler.handleException(e);
         }
     }
+    /***
+     * Description: 上传分片
+     * @param chuckBucketName : bucketName
+     * @param fileMd5 : 文件的md5值
+     * @param inputStream : 文件流
+     * @param totalChunks : 总分片数
+     * @param currentChuckNumber : 当前分片数
+     * return : void
+     * @author : lijie
+     * Update By 2021/12/17 15:18
+     */
+    @Override
+    public void uploadChuck(String chuckBucketName, String fileMd5, InputStream inputStream,Integer totalChunks,
+                            Integer currentChuckNumber) {
+
+    }
 
     /**
      * 文件下载

dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/impl/MinioStorageServiceImpl.java

@@ -1,6 +1,7 @@
 package com.persagy.dmp.file.service.impl;
 
 import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.io.IoUtil;
 import cn.hutool.core.map.MapUtil;
 import cn.hutool.core.util.StrUtil;
 import com.persagy.dmp.common.exception.BusinessException;
@@ -19,6 +20,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Value;
 
+import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -33,9 +35,11 @@ public class MinioStorageServiceImpl implements IFileStorageService, Initializin
 
     private final MinioClient minioClient;
 
-    @Value("${persagy.common.file.minio.bucket.chunk:bdtp-chuck}")
+    @Value("${persagy.common.file.minio.bucket.chunk:bdtp}")
     private String chunkBucKet;
 
+    private static final Long CHUCK_MIN_FILE_SIZE=5242880L;
+
     /**排序*/
     public final static boolean SORT = true;
     /**不排序*/
@@ -63,12 +67,88 @@ public class MinioStorageServiceImpl implements IFileStorageService, Initializin
             if (exists(bucketName,fileName)){
                 return;
             }
+            if (chunkBucKet.equals(bucketName)){
+                // 检查一下分片内容大小
+                // 合并文件之前先检查一下文件是否正确
+                byte[] bytes = IoUtil.readBytes(input);
+                if (bytes.length<CHUCK_MIN_FILE_SIZE){
+                    // 进行合并分片操作
+                    mergeMinFile(bucketName,fileName,bytes);
+                    return;
+                }
+                input = IoUtil.toStream(bytes);
+            }
             minioClient.putObject(PutObjectArgs.builder().bucket(bucketName).object(fileName).
                             stream(input, input.available(), -1).build());
         } catch (Exception e) {
             FileExceptionHandler.handleException(e);
         }
     }
+    /***
+     * Description: 上传分片
+     * @param chuckBucketName : bucketName
+     * @param fileMd5 : 文件的md5值
+     * @param inputStream : 文件流
+     * @param totalChunks : 总分片数
+     * @param currentChuckNumber : 当前分片数
+     * return : void
+     * @author : lijie
+     * Update By 2021/12/17 15:18
+     */
+    @Override
+    public void uploadChuck(String chuckBucketName, String fileMd5, InputStream inputStream,Integer totalChunks,
+                            Integer currentChuckNumber) {
+        byte[] bytes = IoUtil.readBytes(inputStream);
+        // 如果分片大小大于5M则直接上传
+        upload(chuckBucketName,fileMd5+StrUtil.SLASH+currentChuckNumber+FileCommonConst.CHUCK_FILE_SUFFIX,IoUtil.toStream(bytes));
+        // 2021年12月17日15:53:47 暂时不考虑
+//        if (bytes.length>CHUCK_MIN_FILE_SIZE){
+//            // 如果分片大小大于5M则跳过检查
+//            return;
+//        }
+//        int groupSize = (int)Math.ceil(CHUCK_MIN_FILE_SIZE / (bytes.length * 0.1));
+//        int groupNum = (int)Math.ceil(currentChuckNumber / (groupSize * 0.1));
+//        int startChuckNumber = (groupNum-1)*groupSize;
+    }
+
+    /***
+     * Description: 合并小分片
+     * @param bucketName : bucket的名称
+     * @param fileName : 文件名称
+     * @param bytes : 字节内容
+     * return : void
+     * @author : lijie
+     * Update By 2021/12/17 12:23
+     */
+    @SneakyThrows
+    private void mergeMinFile(String bucketName, String fileName, byte[] bytes) {
+        String prefix = fileName.substring(0, fileName.lastIndexOf("/"));
+        List<String> objectNames = listObjectNames(bucketName, prefix, Boolean.TRUE);
+        if (CollUtil.isEmpty(objectNames)){
+            ByteArrayInputStream inputStream = IoUtil.toStream(bytes);
+            minioClient.putObject(PutObjectArgs.builder().bucket(bucketName).object(fileName).
+                    stream(inputStream, inputStream.available(), -1).build());
+            return;
+        }
+        // 开始检查分片,每次都从头到尾检查一遍.防止出现并发上传的分片
+
+
+
+        String lastObjName = objectNames.get(objectNames.size() - 1);
+        StatObjectResponse objectResponse = minioClient
+                .statObject(StatObjectArgs.builder().bucket(bucketName).object(lastObjName).build());
+        if (objectResponse.size()>CHUCK_MIN_FILE_SIZE){
+            ByteArrayInputStream inputStream = IoUtil.toStream(bytes);
+            minioClient.putObject(PutObjectArgs.builder().bucket(bucketName).object(fileName).
+                    stream(inputStream, inputStream.available(), -1).build());
+            return;
+        }
+        InputStream lastObj = minioClient.getObject(GetObjectArgs.builder().bucket(bucketName).object(lastObjName).build());
+        byte[] lastBytes = IoUtil.readBytes(lastObj);
+
+
+
+    }
 
     @Override
     public InputStream download(String bucketName, String fileName) {
@@ -297,7 +377,7 @@ public class MinioStorageServiceImpl implements IFileStorageService, Initializin
         }
         List<String> urlList = new ArrayList<>(chunkCount);
         for (int i = 1; i <= chunkCount; i++){
-            String objectName = objectMd5 + i + ".chunk";
+            String objectName = objectMd5 + i + FileCommonConst.CHUCK_FILE_SUFFIX;
             urlList.add(createUploadUrl(bucketName,objectName,DEFAULT_EXPIRY));
         }
         return urlList;
@@ -320,7 +400,7 @@ public class MinioStorageServiceImpl implements IFileStorageService, Initializin
         if (StrUtil.isBlank(objectMd5)){
             return null;
         }
-        objectMd5 += "/" + partNumber + ".chunk";
+        objectMd5 += "/" + partNumber + FileCommonConst.CHUCK_FILE_SUFFIX;
         return createUploadUrl(bucketName,objectMd5,DEFAULT_EXPIRY);
     }
 

dmp-comp/dmp-file-starter/src/main/java/com/persagy/dmp/file/constant/FileCommonConst.java

@@ -18,4 +18,6 @@ public interface FileCommonConst {
     Integer UPLOAD_PART = 1;
     /** 桶名校验正则 */
     String BUCKET_PATTERN = "^[a-z0-9][a-z0-9\\-]*[a-z0-9]$";
+    /** 分片文件的后缀 */
+    String CHUCK_FILE_SUFFIX = ".chunk";
 }