功能:调整迁移逻辑,加上只迁移物理世界文件的逻辑

dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/dao/FileMapper.java

@@ -49,4 +49,14 @@ public interface FileMapper extends BaseMapper<FileInfo> {
      * Update By lijie 2021/10/23 13:20
      */
     List<FileInfo> getFileInfoByIds(@Param("fileIds") Set<String> fileIds);
+    /***
+     * Description: 获得文件key列表
+     * @param schema : 库名称前缀
+     * @param groupCode : 集团编码
+     * @param extractInfos : 扩展信息点
+     * @return : java.util.Set<java.lang.String>
+     * @author : lijie
+     * Update By 2021/12/16 14:08
+     */
+    String queryFileKeysByInfos(@Param("schema") String schema,@Param("groupCode") String groupCode,@Param("extractInfos") Set<String> extractInfos);
 }

dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/impl/ImageUrlToImageUrlServiceImpl.java

@@ -17,6 +17,7 @@ import com.persagy.dmp.common.constant.CommonConstant;
 import com.persagy.dmp.common.constant.ResponseCode;
 import com.persagy.dmp.common.exception.BusinessException;
 import com.persagy.dmp.file.context.OldFileAppContext;
+import com.persagy.dmp.file.dao.FileMapper;
 import com.persagy.dmp.file.service.CompatibleOldFileService;
 import com.persagy.dmp.file.service.FileMigrateService;
 import lombok.Data;
@@ -40,6 +41,8 @@ import java.util.Set;
 public class ImageUrlToImageUrlServiceImpl implements FileMigrateService {
 
     private final CompatibleOldFileService compatibleOldFileService;
+    private final FileMapper fileMapper;
+
 
     @Override
     public void migrateFiles(Map<String, Object> requestMap) {
@@ -51,6 +54,10 @@ public class ImageUrlToImageUrlServiceImpl implements FileMigrateService {
         final String toSecret = MapUtil.getStr(requestMap, "toSecret");
         final Set<String> fileKeys = MapUtil.get(requestMap, "keys", new TypeReference<Set<String>>() {});
         final boolean deleteOldFileFlag = MapUtil.getBool(requestMap, "deleteOldFileFlag",false);
+        final boolean physicalMigrateFlag = MapUtil.getBool(requestMap, "physicalMigrateFlag",false);
+        final Set<String> extractInfos = MapUtil.get(requestMap, "extractInfos",new TypeReference<Set<String>>() {});
+        final Set<String> groupCodes = MapUtil.get(requestMap, "groupCodes",new TypeReference<Set<String>>() {});
+        final String schema = MapUtil.getStr(requestMap, "schema");
         if(StrUtil.isBlank(fromImageServiceUrl)
                 || StrUtil.isBlank(fromSystemId)
                 || StrUtil.isBlank(toImageServiceUrl)
@@ -63,33 +70,38 @@ public class ImageUrlToImageUrlServiceImpl implements FileMigrateService {
         }
         // 启动迁移 执行时间较长，后台线程运行
         // TODO 需要加分布式锁，并支持查看执行情况
-        ThreadUtil.execute(() -> startMigrate(fromImageServiceUrl,fromSystemId,fromSecret,toImageServiceUrl,toSystemId,toSecret,deleteOldFileFlag,fileKeys));
+        ThreadUtil.execute(() -> startMigrate(fromImageServiceUrl,fromSystemId,fromSecret,toImageServiceUrl,toSystemId,
+                toSecret,deleteOldFileFlag,fileKeys,physicalMigrateFlag,extractInfos,groupCodes,schema));
     }
 
     /***
      * Description: 启动数据迁移
-     * @param fromImageServiceUrl: 来源文件的地址
-     * @param fromSystemId:来源文件的用户id
-     * @param fromSecret:来源文件的用户密码
-     * @param toImageServiceUrl:去向文件的地址
-     * @param toSystemId:去向文件的用户id
-     * @param toSecret:去向文件的用户密码
-     * @param deleteOldFileFlag:是否删除旧文件的标记
-     * @param fileKeys:是否只迁移部分文件key列表
-     * @return : void
+     * @param fromImageServiceUrl : 来源文件的地址
+     * @param fromSystemId :来源文件的用户id
+     * @param fromSecret :来源文件的用户密码
+     * @param toImageServiceUrl :去向文件的地址
+     * @param toSystemId :去向文件的用户id
+     * @param toSecret :去向文件的用户密码
+     * @param deleteOldFileFlag :是否删除旧文件的标记
+     * @param fileKeys :是否只迁移部分文件key列表
+     * @param physicalMigrateFlag: 只迁移物理世界文件标记
+     * @param extractInfos:额外拥有图片key的信息点
+     * @param groupCodes: 集团编码
      * @author : lijie
      * @date :2021/9/9 23:31
      * Update By lijie 2021/9/9 23:31
      */
     private void startMigrate(String fromImageServiceUrl, String fromSystemId, String fromSecret, String toImageServiceUrl,
-                              String toSystemId, String toSecret, boolean deleteOldFileFlag, Set<String> fileKeys){
+                              String toSystemId, String toSecret, Boolean deleteOldFileFlag, Set<String> fileKeys,
+                              Boolean physicalMigrateFlag, Set<String> extractInfos, Set<String> groupCodes,String schema){
         final Map<String,String> uriMap = MapUtil.of("/common/images_list","/common/image_get");
         uriMap.put("/common/files_list","/common/file_get");
         final String urlFormat = "{}{}?systemId={}";
         for (String listSubUrl : uriMap.keySet()) {
             String requestListUrl = StrUtil.format(urlFormat,fromImageServiceUrl,listSubUrl,fromSystemId);
             transferFile(requestListUrl,null,fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,
-                    fileKeys,uriMap.get(listSubUrl),toImageServiceUrl,toSystemId,toSecret);
+                    fileKeys,uriMap.get(listSubUrl),toImageServiceUrl,toSystemId,toSecret,physicalMigrateFlag,
+                    extractInfos,groupCodes,schema);
         }
     }
     /***
@@ -101,9 +113,29 @@ public class ImageUrlToImageUrlServiceImpl implements FileMigrateService {
      */
     private void transferFile(String requestListUrl,String fromKey,String fromImageServiceUrl, String fromSystemId,
                               String fromSecret, Boolean deleteOldFileFlag,Set<String> destFileKeys, String getUrl,
-                              String toImageServiceUrl,String toSystemId,String toSecret) {
+                              String toImageServiceUrl,String toSystemId,String toSecret,
+                              Boolean physicalMigrateFlag, Set<String> extractInfos, Set<String> groupCodes,String schema) {
         final int pageSize = 1000;
         JSONObject param = new JSONObject();
+        // 1.如果仅迁移物理世界的图片数据
+        if (CollUtil.isEmpty(destFileKeys) && physicalMigrateFlag
+                && CollUtil.isNotEmpty(groupCodes)
+                && StrUtil.isNotBlank(schema)){
+            for (String groupCode : groupCodes) {
+                String fileKey = fileMapper.queryFileKeysByInfos(schema,groupCode.toLowerCase(),extractInfos);
+                if (StrUtil.isBlank(fileKey)){
+                    continue;
+                }
+                List<String> fileKeys = JSONArray.parseArray(fileKey, String.class);
+                if (CollUtil.isEmpty(fileKeys)){
+                    continue;
+                }
+                // 指定迁移key列表
+                transferFileInternal(CollUtil.newHashSet(fileKeys),fromImageServiceUrl,fromSystemId,fromSecret,deleteOldFileFlag,getUrl,
+                        toImageServiceUrl,toSystemId,toSecret);
+            }
+            return;
+        }
         // 1.如果指定文件列表不为空则直接迁移指定列表的文件
         if (CollUtil.isNotEmpty(destFileKeys)){
             // 指定迁移key列表
@@ -139,7 +171,8 @@ public class ImageUrlToImageUrlServiceImpl implements FileMigrateService {
             return;
         }
         transferFile(requestListUrl,fileKeys.get(fileKeys.size() - 1),fromImageServiceUrl,fromSystemId,fromSecret,
-                deleteOldFileFlag,destFileKeys,getUrl,toImageServiceUrl,toSystemId,toSecret);
+                deleteOldFileFlag,destFileKeys,getUrl,toImageServiceUrl,toSystemId,toSecret,physicalMigrateFlag,
+                extractInfos,groupCodes,schema);
     }
     /***
      * Description: 根据key列表迁移文件

dmp-cloud/dmp-file/src/main/java/com/persagy/dmp/file/service/impl/ImageUrlToMinioServiceImpl.java

@@ -15,6 +15,7 @@ import com.persagy.dmp.common.constant.CommonConstant;
 import com.persagy.dmp.common.constant.ResponseCode;
 import com.persagy.dmp.common.exception.BusinessException;
 import com.persagy.dmp.file.context.OldFileAppContext;
+import com.persagy.dmp.file.dao.FileMapper;
 import com.persagy.dmp.file.service.CompatibleOldFileService;
 import com.persagy.dmp.file.service.FileMigrateService;
 import lombok.Data;
@@ -38,6 +39,7 @@ import java.util.Set;
 public class ImageUrlToMinioServiceImpl implements FileMigrateService {
 
     private final CompatibleOldFileService compatibleOldFileService;
+    private final FileMapper fileMapper;
 
     @Override
     public void migrateFiles(Map<String, Object> requestMap) {
@@ -47,6 +49,10 @@ public class ImageUrlToMinioServiceImpl implements FileMigrateService {
         final String secret = MapUtil.getStr(requestMap, "secret");
         final String bucketName = MapUtil.getStr(requestMap, "bucketName","persagy");
         final boolean deleteOldFileFlag = MapUtil.getBool(requestMap, "deleteOldFileFlag",false);
+        final boolean physicalMigrateFlag = MapUtil.getBool(requestMap, "physicalMigrateFlag",false);
+        final Set<String> extractInfos = MapUtil.get(requestMap, "extractInfos",new TypeReference<Set<String>>() {});
+        final Set<String> groupCodes = MapUtil.get(requestMap, "groupCodes",new TypeReference<Set<String>>() {});
+        final String schema = MapUtil.getStr(requestMap, "schema");
         if(StrUtil.isBlank(imageServiceUrl)
                 || StrUtil.isBlank(systemId)
                 || StrUtil.isBlank(bucketName)) {
@@ -57,7 +63,8 @@ public class ImageUrlToMinioServiceImpl implements FileMigrateService {
         }
         // 启动迁移 执行时间较长，后台线程运行
         // TODO 需要加分布式锁，并支持查看执行情况
-        ThreadUtil.execute(() -> startMigrate(imageServiceUrl,systemId,secret,bucketName,deleteOldFileFlag,fileKeys));
+        ThreadUtil.execute(() -> startMigrate(imageServiceUrl,systemId,secret,bucketName,deleteOldFileFlag,fileKeys,
+                physicalMigrateFlag,extractInfos,groupCodes,schema));
     }
 
     /***
@@ -73,13 +80,15 @@ public class ImageUrlToMinioServiceImpl implements FileMigrateService {
      * Update By lijie 2021/9/9 23:31
      */
     private void startMigrate(String imageServiceUrl, String systemId, String secret, String bucketName,
-                              Boolean deleteOldFileFlag,Set<String> fileKeys){
+                              Boolean deleteOldFileFlag,Set<String> fileKeys,
+                              Boolean physicalMigrateFlag, Set<String> extractInfos, Set<String> groupCodes,String schema){
         final Map<String,String> uriMap = MapUtil.of("/common/images_list","/common/image_get");
         uriMap.put("/common/files_list","/common/file_get");
         final String urlFormat = "{}{}?systemId={}";
         for (String listSubUrl : uriMap.keySet()) {
             String requestListUrl = StrUtil.format(urlFormat,imageServiceUrl,listSubUrl,systemId);
-            transferFile(requestListUrl,null,imageServiceUrl,systemId,secret,bucketName,deleteOldFileFlag,fileKeys,uriMap.get(listSubUrl));
+            transferFile(requestListUrl,null,imageServiceUrl,systemId,secret,bucketName,deleteOldFileFlag,
+                    fileKeys,uriMap.get(listSubUrl),physicalMigrateFlag, extractInfos,groupCodes,schema);
         }
     }
     /***
@@ -91,9 +100,29 @@ public class ImageUrlToMinioServiceImpl implements FileMigrateService {
      */
     private void transferFile(String requestListUrl,String fromKey,String imageServiceUrl, String systemId,
                               String secret, String bucketName, Boolean deleteOldFileFlag,Set<String> destFileKeys,
-                              String getUrl) {
+                              String getUrl, Boolean physicalMigrateFlag, Set<String> extractInfos,
+                              Set<String> groupCodes,String schema) {
         final int pageSize = 1000;
         JSONObject param = new JSONObject();
+        // 1.如果仅迁移物理世界的图片数据
+        if (CollUtil.isEmpty(destFileKeys) && physicalMigrateFlag
+                && CollUtil.isNotEmpty(groupCodes)
+                && StrUtil.isNotBlank(schema)){
+            for (String groupCode : groupCodes) {
+                String fileKey = fileMapper.queryFileKeysByInfos(schema,groupCode.toLowerCase(),extractInfos);
+                if (StrUtil.isBlank(fileKey)){
+                    continue;
+                }
+                List<String> fileKeys = JSONArray.parseArray(fileKey, String.class);
+                if (CollUtil.isEmpty(fileKeys)){
+                    continue;
+                }
+                // 指定迁移key列表
+                transferFileInternal(CollUtil.newHashSet(fileKeys),imageServiceUrl,systemId, secret,
+                        bucketName,deleteOldFileFlag,getUrl);
+            }
+            return;
+        }
         // 1.如果指定文件列表不为空则直接迁移指定列表的文件
         if (CollUtil.isNotEmpty(destFileKeys)){
             // 指定迁移key列表
@@ -127,7 +156,7 @@ public class ImageUrlToMinioServiceImpl implements FileMigrateService {
             return;
         }
         transferFile(requestListUrl,fileKeys.get(fileKeys.size() - 1),imageServiceUrl,systemId,secret,bucketName,
-                deleteOldFileFlag,destFileKeys,getUrl);
+                deleteOldFileFlag,destFileKeys,getUrl,physicalMigrateFlag, extractInfos,groupCodes,schema);
     }
     /***
      * Description: 根据key列表迁移文件

dmp-cloud/dmp-file/src/main/resources/mapper/FileMapper.xml

@@ -61,5 +61,29 @@
             #{fileId}
         </foreach>
     </select>
+    <select id="queryFileKeysByInfos" resultType="java.lang.String">
+        SELECT
+        <choose>
+            <when test="extractInfos!=null and extractInfos.size()>0">
+                <foreach collection="extractInfos" item="item" open="JSON_EXTRACT(JSON_ARRAYAGG(JSON_EXTRACT(infos, '$.*.key'," close=")),'$[*][*]')" separator=",">
+                    CONCAT('$.',#{item})
+                </foreach>
+            </when>
+            <otherwise>
+                JSON_EXTRACT(JSON_ARRAYAGG(JSON_EXTRACT(infos, '$.*.key')),'$[*][*]')
+            </otherwise>
+        </choose> AS fileKeys
+        FROM ${schema}_${groupCode}.dt_object WHERE
+        <choose>
+            <when test="extractInfos!=null and extractInfos.size()>0">
+                <foreach collection="extractInfos" item="item" open="JSON_EXTRACT(infos, '$.*.key'," close=") IS NOT NULL" separator=",">
+                    CONCAT('$.',#{item})
+                </foreach>
+            </when>
+            <otherwise>
+                JSON_EXTRACT(infos, '$.*.key') IS NOT NULL
+            </otherwise>
+        </choose>;
+    </select>
 </mapper>