活跃用户入redis_多进程

主函数active_user.sh

#!/bin/bash

# 初始化环境
ROOTDIR="/data/active_user"
TMPDIR="${ROOTDIR}/tmp"
LOGDIR="${ROOTDIR}/logs"

mkdir -p $LOGDIR
mkdir -p $TMPDIR

# 初始化日志文件
LOGFILE="${LOGDIR}/active_user.log"
ERRFILE="${LOGDIR}/active_user.err"
[ ! -f $LOGFILE ] && touch $LOGFILE
[ ! -f $ERRFILE ] && touch $ERRFILE
[ -f ${LOGFILE} ] && exec 1>> ${LOGFILE}
[ -f ${ERRFILE} ] && exec 2>> ${ERRFILE}

# 初始化工具
REDIS_IP=127.0.0.1
REDIS_PORT=6379
REDIS_DB=8
REDIS_PASSWD='Muke#0428'
CMD_REDIS="/bin/redis-cli -h $REDIS_IP -p $REDIS_PORT -a $REDIS_PASSWD -n $REDIS_DB "
CMD_CURL="curl -s"

# 初始化redis键值
REDIS_KEY='crm_activity_list'

# 初始化接口
URL_SET_ACCOUNT_CACHE=https://bi.cnabke.com/abke-bi/expose/setAccountCache/
URL_SUCCESS_ACCOUNT_CACHE=https://bi.cnabke.com/abke-bi/expose/successAccountCache/

# 定义通用函数
# crm 生产库
db_crm_online()
{
    export MYSQL_PWD="shmk!@#45"
    local _DBHOST=rr-bp1u6dbk7du9x6407.mysql.rds.aliyuncs.com
    local _DBPORT=3306
    local _DBNAME=crm
    local _DBUSER=shmuker
    local sql=$1
    /usr/bin/mysql -A -N -h $_DBHOST -P $_DBPORT -u $_DBUSER -D $_DBNAME -e "${sql}"
}

# 日志函数
logger()
{
    echo $(date "+%Y%m%d%H%M%S"):$*
}


lock_account_cache()
{
    local account_id=$1
    local localfile="${LOGDIR}/active_user.lock"
    if [ -f $localfile ]; then
        return 1
    fi
    touch $localfile
    if [ $? -ne 0 ]; then
        return 1
    fi
    return 0
}

unlock_account_cache()
{
    local account_id=$1
    local localfile="${LOGDIR}/active_user.lock"
    [ -f "${localfile}" ] && rm -f "${localfile}"
    if [ $? -ne 0 ]; then
        return 1
    fi
    return 0
}

main()
{
    #判断是否加锁
    lock_account_cache
    if [ $? -ne 0 ]; then
        echo "lock is faild!!!"
        return 0
    fi
    #测试数据库是否连接
    db_crm_online "select 1;" &> /dev/null
    if [ $? -ne 0 ]; then
        logger "db:test:failed!"
        unlock_account_cache
        return 1
    fi
    logger "db:test:success!"

    # 获取活跃用户id
    local get_account_info_sql="
        SELECT
            DISTINCT t2.m_account_id
        FROM
        (
            SELECT m_mem_id
            FROM crm.m_login_log
            WHERE tm >= DATE_ADD(NOW(),INTERVAL -7 DAY)
            GROUP BY m_mem_id
            HAVING COUNT(*) > 3
        ) t1
        LEFT JOIN
        (
            SELECT m_mem_id, m_account_id FROM crm.m_mem
        ) t2
        ON ( t1.m_mem_id = t2.m_mem_id )
    "
    # 活跃企业 id 入缓存
    db_crm_online "${get_account_info_sql}" | (
    while read line
    do
        $CMD_REDIS SADD ${REDIS_KEY} $line 1> /dev/null
        if [ $? -ne 0 ]; then
            logger "SADD:${REDIS_KEY}:${line}:failed"
        else
            logger "SADD:${REDIS_KEY}:${line}:success"
        fi
    done
    )

    # 开始操作
    local process_limits=4
    for (( i=1; i<= process_limits; i++ ))
    do
        logger "start:$ROOTDIR/active_user_agent.sh:${i}"
        source  $ROOTDIR/active_user_agent.sh &
    done

    # 监听是否操作完成
    while true
    do
        local ret_cnt=$( $CMD_REDIS scard ${REDIS_KEY} )
        if [ $ret_cnt -eq 0 ]; then
            break
        fi
        logger "listen 5 seconds.[$ret_cnt]"
        sleep 5
    done

    local time_end=$(date "+%Y%m%d%H%M%S")
    $CMD_CURL ${URL_SUCCESS_ACCOUNT_CACHE}${time_end} &> /dev/null
    unlock_account_cache
}

logger "start"
main
logger "end"

调用接口脚本: active_user_agent.sh

#!/bin/bash

# logger()
# {
#     echo $(date "+%Y%m%d%H%M%S"):$*
# }
#
# # 初始化redis连接
# REDIS_IP=127.0.0.1
# REDIS_PORT=6379
# REDIS_DB=8
# REDIS_PASSWD='Muke#0428'
# CMD_REDIS="/bin/redis-cli -h $REDIS_IP -p $REDIS_PORT -a $REDIS_PASSWD -n $REDIS_DB "
#
# REDIS_KEY='crm_activity_list'
# URL_SET_ACCOUNT_CACHE=https://bi.cnabke.com/abke-bi/expose/setAccountCache/
# CMD_CURL="curl -s"

main()
{
    while true
    do
        local ret_cnt=$( $CMD_REDIS scard ${REDIS_KEY} )
        if [ $ret_cnt -eq 0 ]; then
            return 0
        fi
        local ret_m_account_id=$( $CMD_REDIS spop ${REDIS_KEY} )
        if [ ! ${ret_m_account_id} ]; then
            logger "active_user_agent:spop:${REDIS_KEY}:failed:${REDIS_KEY} is empty or not exist!"
            continue
        fi
        logger "active_user_agent:spop:${REDIS_KEY}:${ret_m_account_id}:success!"

        $CMD_CURL ${URL_SET_ACCOUNT_CACHE}${ret_m_account_id} &> /dev/null
        if [ $? -ne 0 ]; then
            logger "active_user_agent:curl:${URL_SET_ACCOUNT_CACHE}${ret_m_account_id}:failed"
            continue
        fi
        logger "active_user_agent:curl:${URL_SET_ACCOUNT_CACHE}${ret_m_account_id}:success"
        sleep 2
    done
}

main $*