dbv2

baseimg/tools/dbv2/Dockerfile

+FROM 192.168.4.4/baseimg/centos:201803291115
+
+###
+### Envs
+###
+
+# Version
+# Check for Updates:
+# https://dev.mysql.com/downloads/repo/yum/
+ENV YUM_REPO_URL="https://dev.mysql.com/get/mysql57-community-release-el7-10.noarch.rpm "
+
+# User/Group
+ENV MY_USER="mysql"
+ENV MY_GROUP="mysql"
+ENV MY_UID="48"
+ENV MY_GID="48"
+
+# Files
+ENV MYSQL_BASE_INCL="/etc/my.cnf.d"
+ENV MYSQL_CUST_INCL1="/etc/mysql/conf.d"
+ENV MYSQL_CUST_INCL2="/etc/mysql/docker-default.d"
+ENV MYSQL_DEF_DAT="/var/lib/mysql"
+ENV MYSQL_DEF_LOG="/var/log/mysql"
+ENV MYSQL_DEF_PID="/var/run/mysqld"
+ENV MYSQL_DEF_SCK="/var/sock/mysqld"
+
+ENV MYSQL_LOG_SLOW="${MYSQL_DEF_LOG}/slow.log"
+ENV MYSQL_LOG_ERROR="${MYSQL_DEF_LOG}/error.log"
+ENV MYSQL_LOG_QUERY="${MYSQL_DEF_LOG}/query.log"
+
+###
+### Install
+###
+RUN groupadd -g ${MY_GID} -r ${MY_GROUP} && \
+	adduser ${MY_USER} -u ${MY_UID} -M -s /sbin/nologin -g ${MY_GROUP}
+
+RUN \
+	yum -y install epel-release && \
+	rpm -ivh ${YUM_REPO_URL} && \
+	yum-config-manager --disable mysql55-community && \
+	yum-config-manager --disable mysql56-community && \
+	yum-config-manager --enable mysql57-community && \
+	yum-config-manager --disable mysql80-community && \
+	yum clean all
+
+RUN yum -y update && yum -y install \
+	mysql-community-server
+
+RUN \
+	yum -y autoremove && \
+	yum clean metadata && \
+	yum clean all && \
+	yum -y install hostname && \
+	yum clean all
+
+
+##
+## Configure
+##
+RUN \
+	rm -rf ${MYSQL_BASE_INCL} && \
+	rm -rf ${MYSQL_CUST_INCL1} && \
+	rm -rf ${MYSQL_CUST_INCL2} && \
+	rm -rf ${MYSQL_DEF_DAT} && \
+	rm -rf ${MYSQL_DEF_SCK} && \
+	rm -rf ${MYSQL_DEF_PID} && \
+	rm -rf ${MYSQL_DEF_LOG} && \
+	\
+	mkdir -p ${MYSQL_BASE_INCL} && \
+	mkdir -p ${MYSQL_CUST_INCL1} && \
+	mkdir -p ${MYSQL_CUST_INCL2} && \
+	mkdir -p ${MYSQL_DEF_DAT} && \
+	mkdir -p ${MYSQL_DEF_SCK} && \
+	mkdir -p ${MYSQL_DEF_PID} && \
+	mkdir -p ${MYSQL_DEF_LOG} && \
+	\
+	chown -R ${MY_USER}:${MY_GROUP} ${MYSQL_BASE_INCL} && \
+	chown -R ${MY_USER}:${MY_GROUP} ${MYSQL_CUST_INCL1} && \
+	chown -R ${MY_USER}:${MY_GROUP} ${MYSQL_CUST_INCL2} && \
+	chown -R ${MY_USER}:${MY_GROUP} ${MYSQL_DEF_DAT} && \
+	chown -R ${MY_USER}:${MY_GROUP} ${MYSQL_DEF_SCK} && \
+	chown -R ${MY_USER}:${MY_GROUP} ${MYSQL_DEF_PID} && \
+	chown -R ${MY_USER}:${MY_GROUP} ${MYSQL_DEF_LOG} && \
+	\
+	chmod 0775 ${MYSQL_BASE_INCL} && \
+	chmod 0775 ${MYSQL_CUST_INCL1} && \
+	chmod 0775 ${MYSQL_CUST_INCL2} && \
+	chmod 0775 ${MYSQL_DEF_DAT} && \
+	chmod 0775 ${MYSQL_DEF_SCK} && \
+	chmod 0775 ${MYSQL_DEF_PID} && \
+	chmod 0775 ${MYSQL_DEF_LOG}
+
+RUN \
+	echo "[client]"                                         > /etc/my.cnf && \
+	echo "socket = ${MYSQL_DEF_SCK}/mysqld.sock"           >> /etc/my.cnf && \
+	\
+	echo "[mysql]"                                         >> /etc/my.cnf && \
+	echo "socket = ${MYSQL_DEF_SCK}/mysqld.sock"           >> /etc/my.cnf && \
+	\
+	echo "[mysqld]"                                        >> /etc/my.cnf && \
+	echo "skip-host-cache"                                 >> /etc/my.cnf && \
+	echo "skip-name-resolve"                               >> /etc/my.cnf && \
+	echo "datadir = ${MYSQL_DEF_DAT}"                      >> /etc/my.cnf && \
+	echo "user = ${MY_USER}"                               >> /etc/my.cnf && \
+	echo "port = 3306"                                     >> /etc/my.cnf && \
+	echo "bind-address = 0.0.0.0"                          >> /etc/my.cnf && \
+	echo "socket = ${MYSQL_DEF_SCK}/mysqld.sock"           >> /etc/my.cnf && \
+	echo "pid-file = ${MYSQL_DEF_PID}/mysqld.pid"          >> /etc/my.cnf && \
+	echo "general_log_file = ${MYSQL_LOG_QUERY}"           >> /etc/my.cnf && \
+	echo "slow_query_log_file = ${MYSQL_LOG_SLOW}"         >> /etc/my.cnf && \
+	echo "log-error = ${MYSQL_LOG_ERROR}"                  >> /etc/my.cnf && \
+	echo "log-bin=/var/lib/mysql/mysql-bin"                >> /etc/my.cnf && \
+	echo "!includedir ${MYSQL_BASE_INCL}/"                 >> /etc/my.cnf && \
+	echo "!includedir ${MYSQL_CUST_INCL1}/"                >> /etc/my.cnf && \
+	echo "!includedir ${MYSQL_CUST_INCL2}/"                >> /etc/my.cnf
+
+
+##
+## Bootstrap Scipts
+##
+COPY ./scripts/docker-entrypoint.sh /
+
+
+##
+## Ports
+##
+EXPOSE 3306
+
+
+##
+## Volumes
+##
+VOLUME /var/lib/mysql
+VOLUME /var/log/mysql
+VOLUME /var/sock/mysqld
+VOLUME /etc/mysql/conf.d
+VOLUME /etc/mysql/docker-default.d
+
+
+##
+## Entrypoint
+##
+ENTRYPOINT ["/docker-entrypoint.sh"]
\ No newline at end of file

baseimg/tools/dbv2/scripts/docker-entrypoint.sh

+#!/bin/sh -eu
+
+###
+### Variables
+###
+DEBUG_COMMANDS=0
+
+
+###
+### Functions
+###
+run() {
+	_cmd="${1}"
+	_debug="0"
+
+	_red="\033[0;31m"
+	_green="\033[0;32m"
+	_reset="\033[0m"
+	_user="$(whoami)"
+
+
+	# If 2nd argument is set and enabled, allow debug command
+	if [ "${#}" = "2" ]; then
+		if [ "${2}" = "1" ]; then
+			_debug="1"
+		fi
+	fi
+
+
+	if [ "${DEBUG_COMMANDS}" = "1" ] || [ "${_debug}" = "1" ]; then
+		printf "${_red}%s \$ ${_green}${_cmd}${_reset}\n" "${_user}"
+	fi
+	sh -c "LANG=C LC_ALL=C ${_cmd}"
+}
+
+log() {
+	_lvl="${1}"
+	_msg="${2}"
+
+	_clr_ok="\033[0;32m"
+	_clr_info="\033[0;34m"
+	_clr_warn="\033[0;33m"
+	_clr_err="\033[0;31m"
+	_clr_rst="\033[0m"
+
+	if [ "${_lvl}" = "ok" ]; then
+		printf "${_clr_ok}[OK]   %s${_clr_rst}\n" "${_msg}"
+	elif [ "${_lvl}" = "info" ]; then
+		printf "${_clr_info}[INFO] %s${_clr_rst}\n" "${_msg}"
+	elif [ "${_lvl}" = "warn" ]; then
+		printf "${_clr_warn}[WARN] %s${_clr_rst}\n" "${_msg}" 1>&2	# stdout -> stderr
+	elif [ "${_lvl}" = "err" ]; then
+		printf "${_clr_err}[ERR]  %s${_clr_rst}\n" "${_msg}" 1>&2	# stdout -> stderr
+	else
+		printf "${_clr_err}[???]  %s${_clr_rst}\n" "${_msg}" 1>&2	# stdout -> stderr
+	fi
+}
+
+
+###
+### Read out MySQL Default config
+###
+get_mysql_default_config() {
+	_key="${1}"
+	mysqld --verbose --help  2>/dev/null  | awk -v key="${_key}" '$1 == key { print $2; exit }'
+}
+
+
+###
+### Set MySQL Custom options
+###
+set_mysql_custom_settings() {
+	_conf_sect="${1}"
+	_mysql_key="${2}"
+	_shell_var="${3}"
+	_extra_val="${4}"	# Extra value to append to _shell_var
+	_conf_file="${5}"
+
+
+	if ! set | grep "^${_shell_var}=" >/dev/null 2>&1; then
+		_mysql_val="$( get_mysql_default_config "${_mysql_key}" )"
+		log "info" "\$${_shell_var} not set. Keeping default: [${_conf_sect}] ${_mysql_key}=${_mysql_val}"
+
+	else
+		_shell_val="$( eval "echo \${${_shell_var}}" )"
+		_value="${_shell_val}${_extra_val}"
+
+		if [ "${_value}" = "" ]; then
+			_mysql_val="$( get_mysql_default_config "${_mysql_key}" )"
+			log "warn" "\$${_shell_var} is empty. Keeping default: [${_conf_sect}] ${_mysql_key}=${_mysql_val}"
+
+		else
+			log "info" "Setting MySQL: [${_conf_sect}] ${_mysql_key}=${_value}"
+
+			# Add file
+			if [ ! -f "${_conf_file}" ]; then
+				run "touch ${_conf_file}"
+			fi
+
+			# Add section
+			if ! grep -q "\[${_conf_sect}\]" "${_conf_file}"; then
+				run "echo '[${_conf_sect}]' >> ${_conf_file}"
+				run "echo '${_mysql_key} = ${_value}' >> ${_conf_file}"
+
+			else
+				run "sed -i'' 's|\[${_conf_sect}\]|\[${_conf_sect}\]\n${_mysql_key} = ${_value}|g' ${_conf_file}"
+			fi
+		fi
+	fi
+}
+
+
+
+
+################################################################################
+# BOOTSTRAP
+################################################################################
+
+if set | grep '^DEBUG_COMPOSE_ENTRYPOINT='  >/dev/null 2>&1; then
+	if [ "${DEBUG_COMPOSE_ENTRYPOINT}" = "1" ]; then
+		DEBUG_COMMANDS=1
+	fi
+fi
+
+
+
+################################################################################
+# ENVIRONMENTAL CHECKS
+################################################################################
+
+
+
+###
+### MySQL Password Options
+###
+if ! set | grep '^MYSQL_ROOT_PASSWORD=' >/dev/null 2>&1; then
+	log "err" "\$MYSQL_ROOT_PASSWORD must be set."
+	exit 1
+fi
+
+
+################################################################################
+# MAIN ENTRY POINT
+################################################################################
+
+
+###
+### Adjust timezone
+###
+
+if ! set | grep '^TIMEZONE='  >/dev/null 2>&1; then
+	log "warn" "\$TIMEZONE not set."
+else
+	if [ -f "/usr/share/zoneinfo/${TIMEZONE}" ]; then
+		# Unix Time
+		log "info" "Setting docker timezone to: ${TIMEZONE}"
+		run "rm /etc/localtime"
+		run "ln -s /usr/share/zoneinfo/${TIMEZONE} /etc/localtime"
+	else
+		log "err" "Invalid timezone for \$TIMEZONE."
+		log "err" "\$TIMEZONE: '${TIMEZONE}' does not exist."
+		exit 1
+	fi
+fi
+log "info" "Docker date set to: $(date)"
+
+
+
+
+###
+### Add custom Configuration
+###
+
+# MYSQL_GENERAL_LOG
+set_mysql_custom_settings "mysqld" "general-log" "MYSQL_GENERAL_LOG" "" "${MYSQL_BASE_INCL}/logging.cnf"
+
+# MYSQL_SOCKET_DIR
+set_mysql_custom_settings "client" "socket" "MYSQL_SOCKET_DIR" "/mysqld.sock" "${MYSQL_BASE_INCL}/socket.cnf"
+set_mysql_custom_settings "mysql"  "socket" "MYSQL_SOCKET_DIR" "/mysqld.sock" "${MYSQL_BASE_INCL}/socket.cnf"
+set_mysql_custom_settings "mysqld" "socket" "MYSQL_SOCKET_DIR" "/mysqld.sock" "${MYSQL_BASE_INCL}/socket.cnf"
+
+# Take care about custom socket directories
+if set | grep "^MYSQL_SOCKET_DIR=" >/dev/null 2>&1; then
+
+	# Create socket directory
+	if [ ! -d "${MYSQL_SOCKET_DIR}" ]; then
+		run "mkdir -p ${MYSQL_SOCKET_DIR}"
+
+	# Delete existing socket file
+	elif [ -f "${MYSQL_SOCKET_DIR}/mysqld.sock" ]; then
+		run "rm -f ${MYSQL_SOCKET_DIR}/mysqld.sock"
+	fi
+
+	# Set socket permission
+	run "chown ${MY_USER}:${MY_GROUP} ${MYSQL_SOCKET_DIR}"
+	run "chmod 0777 ${MYSQL_SOCKET_DIR}"
+fi
+
+
+
+
+################################################################################
+# INSTALLATION
+################################################################################
+
+DB_DATA_DIR="$( get_mysql_default_config "datadir" )"
+
+
+##
+## INSTALLATION
+##
+
+# Fixing permissions
+if [ ! -f "${MYSQL_LOG_QUERY}" ]; then
+	run "touch ${MYSQL_LOG_QUERY}"
+fi
+if [ ! -f "${MYSQL_LOG_SLOW}" ]; then
+	run "touch ${MYSQL_LOG_SLOW}"
+fi
+if [ ! -f "${MYSQL_LOG_ERROR}" ]; then
+	run "touch ${MYSQL_LOG_ERROR}"
+fi
+
+run "chown -R ${MY_USER}:${MY_GROUP} ${DB_DATA_DIR}"
+run "chown -R ${MY_USER}:${MY_GROUP} ${MYSQL_DEF_DAT}"
+run "chown -R ${MY_USER}:${MY_GROUP} ${MYSQL_DEF_LOG}"
+run "chown -R ${MY_USER}:${MY_GROUP} ${MYSQL_DEF_PID}"
+run "chown -R ${MY_USER}:${MY_GROUP} ${MYSQL_DEF_SCK}"
+
+run "chmod 0775 ${DB_DATA_DIR}"
+run "chmod 0775 ${MYSQL_DEF_DAT}"
+run "chmod 0775 ${MYSQL_DEF_LOG}"
+run "chmod 0775 ${MYSQL_DEF_PID}"
+run "chmod 0775 ${MYSQL_DEF_SCK}"
+
+run "find ${MYSQL_DEF_LOG}/ -type f -exec chmod 0664 {} \;"
+
+# Directory already exists and has content (other thab '.' and '..')
+if [ -d "${DB_DATA_DIR}/mysql" ] && [ "$( ls -A "${DB_DATA_DIR}/mysql" )" ]; then
+	log "info" "Found existing data directory. MySQL already setup."
+
+else
+
+	log "info" "No existing MySQL data directory found. Setting up MySQL for the first time."
+
+	# Create datadir if not exist yet
+	if [ ! -d "${DB_DATA_DIR}" ]; then
+		log "info" "Creating empty data directory in: ${DB_DATA_DIR}."
+		run "mkdir -p ${DB_DATA_DIR}"
+		run "chown -R ${MY_USER}:${MY_GROUP} ${DB_DATA_DIR}"
+		run "chmod 0777 ${MY_USER}:${MY_GROUP} ${DB_DATA_DIR}"
+	fi
+
+
+	# Install Database
+	run "mysqld --initialize-insecure --datadir=${DB_DATA_DIR} --user=${MY_USER}"
+
+
+	# Start server
+	run "mysqld --skip-networking &"
+
+
+	# Wait at max 60 seconds for it to start up
+	i=0
+	max=60
+	while [ $i -lt $max ]; do
+		if echo 'SELECT 1' |  mysql --protocol=socket -uroot  > /dev/null 2>&1; then
+			break
+		fi
+		log "info" "Initializing ..."
+		sleep 1s
+		i=$(( i + 1 ))
+	done
+
+
+	# Get current pid
+	pid="$(pgrep mysqld | head -1)"
+	if [ "${pid}" = "" ]; then
+		log "err" "Could not find running MySQL PID."
+		log "err" "MySQL init process failed."
+		exit 1
+	fi
+
+
+	# Bootstrap MySQL
+	log "info" "Setting up root user permissions."
+	echo "DELETE FROM mysql.user ;" | mysql --protocol=socket -uroot
+	echo "CREATE USER 'root'@'%' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ;" | mysql --protocol=socket -uroot
+	echo "GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION ;" | mysql --protocol=socket -uroot
+	echo "DROP DATABASE IF EXISTS test ;" | mysql --protocol=socket -uroot
+	echo "FLUSH PRIVILEGES ;" | mysql --protocol=socket -uroot
+
+
+	# Shutdown MySQL
+	log "info" "Shutting down MySQL."
+	run "kill -s TERM ${pid}"
+	i=0
+	max=60
+	while [ $i -lt $max ]; do
+		if ! pgrep mysqld >/dev/null 2>&1; then
+			break
+		fi
+		sleep 1s
+		i=$(( i + 1 ))
+	done
+
+
+	# Check if it is still running
+	if pgrep mysqld >/dev/null 2>&1; then
+		log "err" "Unable to shutdown MySQL server."
+		log "err" "MySQL init process failed."
+		exit 1
+	fi
+	log "info" "MySQL successfully installed."
+
+fi
+
+
+###
+### Start
+###
+log "info" "Starting $(mysqld --version)"
+exec mysqld &
+
+
+### 
+### sync
+###
+USER='qa'
+PASSWORD='qatest'
+
+count=1
+db_is_running=0
+
+export NAMESPACE=`cat /var/run/secrets/kubernetes.io/serviceaccount/namespace`
+
+until [[ $count -eq 300 ]] || [[ $db_is_running -eq 1 ]]
+do
+  let count+=1
+  echo "count is $count"
+  mysqladmin -h db -u$USER -p$PASSWORD processlist
+  if [ $? -eq 0 ]
+    then
+      echo "db is running"
+      db_is_running=1
+    else
+      echo "db is not running"
+      sleep 1
+  fi
+done
+
+if [[ $db_is_running -eq 1 ]]
+  then
+    echo "db is running!!! start execution sync job"
+	
+    echo "start replace db domain"
+    DB_FILE=dump.sql
+    HOST=$DB_SERVICE_HOST
+    PORT='3306'
+    HTTPS='true'
+
+    echo "HOST=$HOST
+    PORT=$PORT
+    USER=$USER
+    PASSWORD=$PASSWORD
+    HTTPS=$HTTPS
+    NAMESPACE=$NAMESPACE"
+
+    EXCLUDED_TABLES=(
+    clotho.JOB_EXECUTION_LOG
+    clotho.JOB_STATUS_TRACE_LOG
+    notify.JOB_EXECUTION_LOG
+    notify.JOB_STATUS_TRACE_LOG
+    xyqb.JOB_EXECUTION_LOG
+    xyqb.JOB_STATUS_TRACE_LOG
+    xyqb.t_phonenum
+    mall.jd_address
+    mall.jd_brand
+    mall.jd_product_sku_image
+    mall.jd_property
+    mall.jd_sku
+    mall.jd_sku_detail_image
+    mall.jd_sku_specs
+    mall.spu_image
+    mall.sale_product_detail
+    mall.product_nature_value_bak
+    mall.product_sku
+    mall.product_sku_image
+    mall.product_specification
+    mall.product_spu
+    )
+
+    IGNORED_TABLES_STRING=''
+    for TABLE in "${EXCLUDED_TABLES[@]}"
+    do :
+       IGNORED_TABLES_STRING+=" --ignore-table=${TABLE}"
+    done
+
+    echo "mysqldump --host=${HOST} --port=${PORT} --user=${USER} --password=${PASSWORD} --all-databases --no-create-info ${IGNORED_TABLES_STRING} > ${DB_FILE}"
+    mysqldump --host=${HOST} --port=${PORT} --user=${USER} --password=${PASSWORD} --all-databases ${IGNORED_TABLES_STRING} >> ${DB_FILE}
+    echo "sed file"
+    if [[ $HTTPS = true ]];then
+    	sed -i "s#\.xyqb\.com#-${NAMESPACE}\.q-gp\.com#g; s#\.quantgroup\.cn#-${NAMESPACE}\.q-gp\.com#g; s#-(\w*)\.q-gp\.com#-${NAMESPACE}\.q-gp\.com#g" ${DB_FILE}
+    else
+    	sed -i "s#\.xyqb\.com#-${NAMESPACE}\.q-gp\.com#g; s#\.quantgroup\.cn#-${NAMESPACE}\.q-gp\.com#g; s#-(\w*)\.q-gp\.com#-${NAMESPACE}\.q-gp\.com#g; s#\https#http#g" ${DB_FILE}
+    fi
+
+    echo "mysql dump file"
+    mysql --host=${HOST} --port=${PORT}  --user=${USER} --password=${PASSWORD} < ${DB_FILE}
+    
+  else
+    echo "db is not running!!! Do not execution replace job"
+fi
+
+tail -f /var/log/mysql