From 2a0eae9c7d20887437c4304837eec0a7f408d5a6 Mon Sep 17 00:00:00 2001
From: Chris Conner <cconner@cloudera.com>
Date: Tue, 31 Mar 2015 14:31:24 -0400
Subject: [PATCH] Memory restart Tools

---
 tools/memory_monitor/hue_mem_cron.sh    | 63 +++++++++++++++++++++++
 tools/memory_monitor/hue_mem_cron_cm.sh | 90 +++++++++++++++++++++++++++++++++
 2 files changed, 153 insertions(+)
 create mode 100644 tools/memory_monitor/hue_mem_cron.sh
 create mode 100644 tools/memory_monitor/hue_mem_cron_cm.sh

diff --git a/tools/memory_monitor/hue_mem_cron.sh b/tools/memory_monitor/hue_mem_cron.sh
new file mode 100644
index 0000000..c2c094f
--- /dev/null
+++ b/tools/memory_monitor/hue_mem_cron.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+#Cron script designed to kill Hue instances that are utilizing too much memory.
+#If using CM, then configure CM to restart the Hue process when it dies and
+#then Hue will restart after this script kills it.  Otherwise, modify this script
+#to start Hue again.
+
+KILL_ME=5000  #This is the number of MB at which it will kill.
+             #Starting with 5000(5gb)
+VERBOSE=true #true then this writes out the proc info each time it runs, leave blank
+             #to only write out when we kill the process
+LOG_FILE=/var/log/hue_mem_cron.log
+ROTATE_SIZE=10 #MB before rotating, size in MB before rotating log to .1, we only keep
+               #2 log files, so 20MB max
+
+main()
+{
+#  Command to get memory usage.  This will handle multiple Hue instances per server, so 
+#  this command grabs the highest memory Hue process at the time of running and 
+#  kills it.  Then the next run it'll get the next process.
+PS_COMMAND=`ps aux | grep [r]uncherrypyserver | awk '{print $6" "$2" "$3" "$12}'`
+MEM=`echo ${PS_COMMAND} | awk '{print $1}'`
+MEM_MB=`expr ${MEM} / 1024`
+PID=`echo ${PS_COMMAND} | awk '{print $2}'`
+CPU=`echo ${PS_COMMAND} | awk '{print $3}'`
+PROC=`echo ${PS_COMMAND} | awk '{print $4}'`
+DATE=`date '+%Y%m%d-%H%M'`
+
+if [[ -f ${LOG_FILE} ]]
+then
+   LOG_SIZE=`du -sm ${LOG_FILE} | awk '{print $1}'`
+   if [ ${LOG_SIZE} -gt ${ROTATE_SIZE} ]
+   then
+      mv ${LOG_FILE} ${LOG_FILE}.1
+   fi
+fi
+
+debug "${DATE} - PID: ${PID} - CPU: ${CPU} - MEM: ${MEM} - MEM_MB: ${MEM_MB} - PROC: ${PROC}"
+
+if [ ${MEM_MB} -gt ${KILL_ME} ]
+then
+   echo "${DATE} - Killing Hue Process: Too much memory: ${MEM_MB} : PID: ${PID}" >> ${LOG_FILE}
+   kill ${PID}
+   sleep 30
+   PID2=`ps aux | grep [r]uncherrypyserver | awk '{print $2}'`
+   if [[ ${PID} == ${PID2} ]]
+   then
+      kill -9 ${PID}
+   fi
+fi
+}
+
+debug()
+{
+
+  if [[ ! -z $VERBOSE ]]
+  then
+    echo "$1" >> ${LOG_FILE}
+  fi
+
+}
+
+main "$@"
diff --git a/tools/memory_monitor/hue_mem_cron_cm.sh b/tools/memory_monitor/hue_mem_cron_cm.sh
new file mode 100644
index 0000000..54b5e7d
--- /dev/null
+++ b/tools/memory_monitor/hue_mem_cron_cm.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+#This script will check CM to see if Hue is using too much memory
+#then will restart if necessary.
+
+CM_HOSTNAME="cdh53-1.qa.test.com"
+CM_PORT="7180"
+CM_USERNAME="admin"
+CM_PASSWORD="admin"
+KILL_ME=5000  #This is the number of MB at which it will kill.
+             #Starting with 5000(5gb)
+VERBOSE=true #true then this writes out the proc info each time it runs, leave blank
+             #to only write out when we kill the process
+LOG_FILE=/var/log/hue_mem_cron.log
+ROTATE_SIZE=10 #MB before rotating, size in MB before rotating log to .1, we only keep
+               #2 log files, so 20MB max
+TMP_LOCATION=/tmp/hue_mem_cron
+
+main()
+{
+DATE=`date '+%Y%m%d-%H%M'`
+YEAR=`date '+%Y'`
+MONTH=`date '+%m'`
+DAY=`date '+%d'`
+HOUR=`date '+%H'`
+MINUTE=`date '+%M'`
+YEAR_PRIOR=`date --date='1 minutes ago' '+%Y'`
+MONTH_PRIOR=`date --date='1 minutes ago' '+%m'`
+DAY_PRIOR=`date --date='1 minutes ago' '+%d'`
+HOUR_PRIOR=`date --date='1 minutes ago' '+%H'`
+MINUTE_PRIOR=`date --date='1 minutes ago' '+%M'`
+MEM_JSON_FILE=${TMP_LOCATION}/mem.json
+MB_BYTES="1048576"
+
+mkdir -p ${TMP_LOCATION}
+
+if [[ -f ${LOG_FILE} ]]
+then
+   LOG_SIZE=`du -sm ${LOG_FILE} | awk '{print $1}'`
+   if [ ${LOG_SIZE} -gt ${ROTATE_SIZE} ]
+   then
+      mv ${LOG_FILE} ${LOG_FILE}.1
+   fi
+fi
+
+MEM_API_URL="/api/v6/timeseries?query=select+mem_rss+where+roleType+%3D+HUE_SERVER&contentType=application%2Fjson&from=${YEAR_PRIOR}-${MONTH_PRIOR}-${DAY_PRIOR}T${HOUR_PRIOR}%3A${MINUTE_PRIOR}%3A00.000Z&to=${YEAR}-${MONTH}-${DAY}T${HOUR}%3A${MINUTE}%3A00.000Z"
+#Get memory usage for all Hue roles:
+curl -X GET -u ${CM_USERNAME}:${CM_PASSWORD} -i -o ${MEM_JSON_FILE} "http://${CM_HOSTNAME}:${CM_PORT}${MEM_API_URL}"
+
+while read -r LINE
+do
+   if  [[ ${LINE} =~ .*clusterName* ]]
+   then
+      CLUSTERNAME=`echo ${LINE} | awk -F\" '{print $4}'`
+   fi
+   if  [[ ${LINE} =~ .*serviceName* ]]
+   then
+      SERVICENAME=`echo ${LINE} | awk -F\" '{print $4}'`
+   fi
+   if  [[ ${LINE} =~ .*roleName* ]]
+   then
+      ROLENAME=`echo ${LINE} | awk -F\" '{print $4}'`
+   fi
+   if  [[ ${LINE} =~ .*value* ]]
+   then
+      MEM=`echo ${LINE} | awk '{print $3}' | awk -F, '{print $1}'`
+      MEM=`printf "%.f" $MEM` # convert from scientific to decimal
+      MEM_MB=`expr ${MEM} / ${MB_BYTES}`
+      debug "${DATE} - ROLENAME: ${ROLENAME} - MEM: ${MEM} - MEM_MB: ${MEM_MB}" 
+      if [ ${MEM_MB} -gt ${KILL_ME} ]
+      then
+         echo "${DATE} - Restart the Hue Process: Too much memory: ${MEM_MB} : ROLENAME: ${ROLENAME}" >> ${LOG_FILE}
+         RESTART_API_URL="/api/v8/clusters/${CLUSTERNAME}/services/${SERVICENAME}/roleCommands/restart"
+         curl -X POST -u ${CM_USERNAME}:${CM_PASSWORD} -i -H "content-type:application/json" -d "{\"items\" : [\"${ROLENAME}\"]}" "http://${CM_HOSTNAME}:${CM_PORT}${RESTART_API_URL}"
+         exit 0
+      fi
+   fi
+done < <(cat ${MEM_JSON_FILE})
+}
+
+debug()
+{
+
+  if [[ ! -z $VERBOSE ]]
+  then
+    echo "$1" >> ${LOG_FILE}
+  fi
+
+}
+
+main "$@"
-- 
1.9.5 (Apple Git-50.3)