saga-bi/example/function/sqoop_func.sh

#! /bin/bash

. /opt/app/bi-app/example/config/config.sh

if [ -n "$2" ] ;then
    echo "如果是输入的日期按照取输入日期"
    do_date=$2
else
    echo "====没有输入数据的日期，取当前时间的前一天===="
    do_date=`date -d yesterday +"%Y-%m-%d"`
fi

url=$MYSQL_URL
username=$MYSQL_USER
password=$MYSQL_PASSWORD


## 无分区
import_no_partition() {
  sqoop import  \
  -D mapred.job.queue.name=hive  \
  --connect $1 \
  --username $2 \
  --password $3 \
  --target-dir $4$7   \
  --delete-target-dir \
  --columns "$5"  \
  --query "$6  and  \$CONDITIONS" \
  --num-mappers 1 \
  --hive-drop-import-delims \
  --fields-terminated-by '\001' \
  --compress \
  --compression-codec lzo \
  --hive-import \
  --hive-database saga_dw \
  --hive-table $7 \
  --hive-overwrite \
  --null-string '\\N' \
  --null-non-string '\\N'
}

dim_date(){
  import_no_partition $url $username $password /warehouse/saga_dw/dim/tmp/ "date_id, week_of_year, day_of_year, day_of_mouth, type, date_type, index_work_day_of_month, week_day, week_day_cn, week_day_en, current_year, current_quarter, year_quarter, mouth, year_mouth, is_first_day_mouth, is_last_day_mouth, type_des" "select date_id, week_of_year, day_of_year, day_of_mouth, type, date_type, index_work_day_of_month, week_day, week_day_cn, week_day_en, current_year, current_quarter, year_quarter, mouth, year_mouth, is_first_day_mouth, is_last_day_mouth, type_des from dim_date where 1 = 1 " dim_date_day
}

dim_office_meter(){
  import_no_partition $url $username $password /warehouse/saga_dw/dim/tmp/ "id, meter, branch_code, branch_name, branch_type, branch_type_name, parent_code, use_range_type, use_range" "select id, meter, branch_code, branch_name, branch_type, branch_type_name, parent_code, use_range_type, use_range from dim_office_meter where 1 = 1 " dim_office_meter
}



case $1 in
"all")
  dim_date
  dim_office_meter
  ;;
"dim_date")
  dim_date
  ;;
"dim_office_meter")
  dim_office_meter
  ;;
esac