InfosTableRepair/repairInfoTable/src/main/java/com/sagacloud/RepairTable.java

package com.sagacloud;
/*
 * Author: Jxing
 * Create Time: 2019/3/12
 */
import com.alibaba.fastjson.*;
import com.google.common.base.Strings;
import com.sagacloud.hbase.HbaseScanIndex;
import com.sagacloud.hbase.ZillionOpUtil;
import com.sagacloud.pojo.HBaseTable;
import com.sagacloud.hbase.HBaseScanUtil;
import com.sagacloud.pojo.InfosObj;
import com.zillion.database.agent.ZillionAgent;
import com.zillion.util.table.HTableUtil;
import com.zillion.util.table.ZillionTableIndex;
import com.zillion.util.table.ZillionTableSchema;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;

import java.io.IOException;
import java.util.*;

public class RepairTable {

    public static void main(String[] args) {
        args = new String[]{"-b", "physical_world_v3_226", "-p", "1101080001"};
        HBaseTable tableInfo = new HBaseTable();
        boolean complete = HBaseTable.getInfo(args, tableInfo);
        if(!complete){
            System.out.println("参数错误, -b 数据库名, -p 项目号(不带Pj)");
            return;
        }
        ZillionAgent agent = ZillionAgentProvider.GetAgent();
        ResultScanner scanner = null;
        JSONObject criteria = new JSONObject();
        if(tableInfo.getProjectId() != null && tableInfo.getProjectId().length() > 0){
            criteria.put("project_id", tableInfo.getProjectId());
        }
        ZillionTableSchema schema = agent.container.repositoryMap.get(tableInfo.getDatabase()).GetTable(tableInfo.getDatatable());
        ZillionTableIndex index = HbaseScanIndex.zillionTableIndex(schema, criteria, null);
        try {
            scanner = getScanner(agent, tableInfo, criteria);
        } catch (Exception e) {
            System.out.println(e.getMessage());
        }
        fixData(scanner, index, schema, tableInfo);
        agent.Stop();
    }

    private static void fixData(ResultScanner scanner, ZillionTableIndex index, ZillionTableSchema schema, HBaseTable tableInfo) {
        if(scanner == null)
            return;
        // key --> objId + infoId , value -->  [ info_value, ArrayList<JSONObjectRow> ]
        Map<String, ArrayList<InfosObj>> fixMap = new LinkedHashMap<>();
        ArrayList<InfosObj> deleteArr = new ArrayList<>();
        try {
            String curProjectId = "aaa";
            while(true) {
                Result[] rs = scanner.next(2000);
                if (rs == null || rs.length == 0) {
                    break;
                }
                for (int i = 0; i < rs.length; ++i) {
                    if (HTableUtil.valid(schema, rs[i].listCells())) {
                        JSONObject resultObject = HBaseScanUtil.Convert(schema, index, rs[i]);
                        if (!resultObject.containsKey("obj_type") || !resultObject.containsKey("obj_id")
                                || !resultObject.containsKey("info_id") || !resultObject.containsKey("time")) {
                            throw new Exception("该表不是数据平台3的obj_infos表");
                        }
                        InfosObj infosObj = resultObject.toJavaObject(InfosObj.class);
                        String idKey = String.join("", infosObj.getProject_id(), infosObj.getObj_type(), infosObj.getObj_id(), infosObj.getInfo_id());
                        putToFixMap(idKey, infosObj, fixMap);
                        if(!(curProjectId == null && infosObj.getProject_id() == null) &&
                                (curProjectId == null || infosObj.getProject_id() == null || !curProjectId.equals(infosObj.getProject_id()))){
                            curProjectId = infosObj.getProject_id();
                            System.out.println("正在处理项目: " + curProjectId);
                            continue;
                        }
                    }
                }
                putToDeleteArr(fixMap, deleteArr);
                if(deleteArr.size() > 2000){
                    ZillionOpUtil.delete(deleteArr, tableInfo);
                    System.out.println("正在修复 " + curProjectId);
                }
            }
            ZillionOpUtil.delete(deleteArr, tableInfo);
        } catch (Exception e) {
            System.out.println(e.getMessage());
            return;
        }finally {
            scanner.close();
        }
        System.out.println("已完成");
    }

    private static void putToDeleteArr(Map<String, ArrayList<InfosObj>> fixMap, ArrayList<InfosObj> deleteArr) {
        String lastKey = null;
        for(String idKey : fixMap.keySet()){
            lastKey = idKey;
            ArrayList<InfosObj> arr = fixMap.get(idKey);
            boolean isFirst = true;
            Object lastValue = null;
            for(int i = 0; i < arr.size(); ++i){
                InfosObj single = arr.get(i);
                if(isFirst){
                    lastValue = single.getObjInfoValue();
                    isFirst = false;
                    continue;
                }
                if(single.getObjInfoValue() == null){
                    if(lastValue == null) {
                        // remove
                        deleteArr.add(single);
                        arr.remove(i);
                        --i;
                        continue;
                    }else{
                        lastValue = single.getObjInfoValue();
                    }
                }else if(single.getObjInfoValue().equals(lastValue)){
                    // remove
                    deleteArr.add(single);
                    arr.remove(i);
                    --i;
                    continue;
                }else{
                    lastValue = single.getObjInfoValue();
                }

            }
        }
        if(lastKey == null ){
            return;
        }
        Iterator<String> iter = fixMap.keySet().iterator();
        while(iter.hasNext()) {
            String key = iter.next();
            if (!key.equals(lastKey)) {
                iter.remove();
            }
        }
    }

    private static void putToFixMap(String idKey, InfosObj resultObject, Map<String, ArrayList<InfosObj>> fixMap) {
        if(fixMap.containsKey(idKey)){
            fixMap.get(idKey).add(resultObject);
        }else{
            ArrayList<InfosObj> arr = new ArrayList<>();
            fixMap.put(idKey, arr);
            arr.add(resultObject);
        }
    }


    private static ResultScanner getScanner(ZillionAgent agent, HBaseTable tableInfo, JSONObject jsonObject) throws Exception {
        return HBaseScanUtil.getScan(agent, tableInfo.getDatabase(), tableInfo.getDatatable(), tableInfo.getDatatable(), jsonObject, null);
    }
}