From dfed3dcb068b68af8f6daea526013ee6558c79be Mon Sep 17 00:00:00 2001 From: mt Date: Tue, 18 Feb 2025 16:10:27 +0300 Subject: [PATCH] Added final changes to repo --- README.md | 10 + docker-compose.yml | 36 ++++ entrypoint.sh | 416 ++++++++++++++++++++++++++++++++++++ scripts/backup_databases.sh | 43 ++++ scripts/my.cnf | 90 ++++++++ scripts/old_qa | 33 +++ scripts/orna_conf | 90 ++++++++ 7 files changed, 718 insertions(+) create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100755 entrypoint.sh create mode 100644 scripts/backup_databases.sh create mode 100644 scripts/my.cnf create mode 100644 scripts/old_qa create mode 100644 scripts/orna_conf diff --git a/README.md b/README.md new file mode 100644 index 0000000..af937d5 --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +# mysql + + +## Getting started + +``` + docker compose up -d + +``` + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5b16b7d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,36 @@ +version: '2.2' +services: + db: + image: mysql:8.0.36 + container_name: mysql_db + restart: always + environment: + MYSQL_DATABASE: 'database' + MYSQL_USER: 'machinetalk' + MYSQL_PASSWORD: 'xM9A4!exJbQNhqJ' + MYSQL_ROOT_PASSWORD: 'xM9A4!exJbQNhqJ' + MYSQL_ROOT_HOST: '%' + ports: + - '172.20.64.140:3306:3306' + expose: + - '3306' + volumes: + - mysql-data:/var/lib/mysql + - ./scripts/my.cnf:/etc/mysql/my.cnf + ulimits: + nofile: + soft: 65536 + hard: 65536 + networks: + - portal_qa + +volumes: + mysql-data: + driver: local + driver_opts: + type: none + device: /data/container_volumes/mysql-data + o: bind +networks: + portal_qa: + external: true diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..8cb17c4 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,416 @@ +#!/bin/bash +set -eo pipefail +shopt -s nullglob + +# logging functions +mysql_log() { + local type="$1"; shift + # accept argument string or stdin + local text="$*"; if [ "$#" -eq 0 ]; then text="$(cat)"; fi + local dt; dt="$(date --rfc-3339=seconds)" + printf '%s [%s] [Entrypoint]: %s\n' "$dt" "$type" "$text" +} +mysql_note() { + mysql_log Note "$@" +} +mysql_warn() { + mysql_log Warn "$@" >&2 +} +mysql_error() { + mysql_log ERROR "$@" >&2 + exit 1 +} + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + mysql_error "Both $var and $fileVar are set (but are exclusive)" + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# usage: docker_process_init_files [file [file [...]]] +# ie: docker_process_init_files /always-initdb.d/* +# process initializer files, based on file extensions +docker_process_init_files() { + # mysql here for backwards compatibility "${mysql[@]}" + mysql=( docker_process_sql ) + + echo + local f + for f; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + mysql_note "$0: running $f" + "$f" + else + mysql_note "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) mysql_note "$0: running $f"; docker_process_sql < "$f"; echo ;; + *.sql.bz2) mysql_note "$0: running $f"; bunzip2 -c "$f" | docker_process_sql; echo ;; + *.sql.gz) mysql_note "$0: running $f"; gunzip -c "$f" | docker_process_sql; echo ;; + *.sql.xz) mysql_note "$0: running $f"; xzcat "$f" | docker_process_sql; echo ;; + *.sql.zst) mysql_note "$0: running $f"; zstd -dc "$f" | docker_process_sql; echo ;; + *) mysql_warn "$0: ignoring $f" ;; + esac + echo + done +} + +# arguments necessary to run "mysqld --verbose --help" successfully (used for testing configuration validity and for extracting default/configured values) +_verboseHelpArgs=( + --verbose --help + --log-bin-index="$(mktemp -u)" # https://github.com/docker-library/mysql/issues/136 +) + +mysql_check_config() { + local toRun=( "$@" "${_verboseHelpArgs[@]}" ) errors + if ! errors="$("${toRun[@]}" 2>&1 >/dev/null)"; then + mysql_error $'mysqld failed while attempting to check config\n\tcommand was: '"${toRun[*]}"$'\n\t'"$errors" + fi +} + +# Fetch value from server config +# We use mysqld --verbose --help instead of my_print_defaults because the +# latter only show values present in config files, and not server defaults +mysql_get_config() { + local conf="$1"; shift + "$@" "${_verboseHelpArgs[@]}" 2>/dev/null \ + | awk -v conf="$conf" '$1 == conf && /^[^ \t]/ { sub(/^[^ \t]+[ \t]+/, ""); print; exit }' + # match "datadir /some/path with/spaces in/it here" but not "--xyz=abc\n datadir (xyz)" +} + +# Ensure that the package default socket can also be used +# since rpm packages are compiled with a different socket location +# and "mysqlsh --mysql" doesn't read the [client] config +# related to https://github.com/docker-library/mysql/issues/829 +mysql_socket_fix() { + local defaultSocket + defaultSocket="$(mysql_get_config 'socket' mysqld --no-defaults)" + if [ "$defaultSocket" != "$SOCKET" ]; then + ln -sfTv "$SOCKET" "$defaultSocket" || : + fi +} + +# Do a temporary startup of the MySQL server, for init purposes +docker_temp_server_start() { + # For 5.7+ the server is ready for use as soon as startup command unblocks + if ! "$@" --daemonize --skip-networking --default-time-zone=SYSTEM --socket="${SOCKET}"; then + mysql_error "Unable to start server." + fi +} + +# Stop the server. When using a local socket file mysqladmin will block until +# the shutdown is complete. +docker_temp_server_stop() { + if ! mysqladmin --defaults-extra-file=<( _mysql_passfile ) shutdown -uroot --socket="${SOCKET}"; then + mysql_error "Unable to shut down server." + fi +} + +# Verify that the minimally required password settings are set for new databases. +docker_verify_minimum_env() { + if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" -a -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then + mysql_error <<-'EOF' + Database is uninitialized and password option is not specified + You need to specify one of the following as an environment variable: + - MYSQL_ROOT_PASSWORD + - MYSQL_ALLOW_EMPTY_PASSWORD + - MYSQL_RANDOM_ROOT_PASSWORD + EOF + fi + + # This will prevent the CREATE USER from failing (and thus exiting with a half-initialized database) + if [ "$MYSQL_USER" = 'root' ]; then + mysql_error <<-'EOF' + MYSQL_USER="root", MYSQL_USER and MYSQL_PASSWORD are for configuring a regular user and cannot be used for the root user + Remove MYSQL_USER="root" and use one of the following to control the root user password: + - MYSQL_ROOT_PASSWORD + - MYSQL_ALLOW_EMPTY_PASSWORD + - MYSQL_RANDOM_ROOT_PASSWORD + EOF + fi + + # warn when missing one of MYSQL_USER or MYSQL_PASSWORD + if [ -n "$MYSQL_USER" ] && [ -z "$MYSQL_PASSWORD" ]; then + mysql_warn 'MYSQL_USER specified, but missing MYSQL_PASSWORD; MYSQL_USER will not be created' + elif [ -z "$MYSQL_USER" ] && [ -n "$MYSQL_PASSWORD" ]; then + mysql_warn 'MYSQL_PASSWORD specified, but missing MYSQL_USER; MYSQL_PASSWORD will be ignored' + fi +} + +# creates folders for the database +# also ensures permission for user mysql of run as root +docker_create_db_directories() { + local user; user="$(id -u)" + + local -A dirs=( ["$DATADIR"]=1 ) + local dir + dir="$(dirname "$SOCKET")" + dirs["$dir"]=1 + + # "datadir" and "socket" are already handled above (since they were already queried previously) + local conf + for conf in \ + general-log-file \ + keyring_file_data \ + pid-file \ + secure-file-priv \ + slow-query-log-file \ + ; do + dir="$(mysql_get_config "$conf" "$@")" + + # skip empty values + if [ -z "$dir" ] || [ "$dir" = 'NULL' ]; then + continue + fi + case "$conf" in + secure-file-priv) + # already points at a directory + ;; + *) + # other config options point at a file, but we need the directory + dir="$(dirname "$dir")" + ;; + esac + + dirs["$dir"]=1 + done + + mkdir -p "${!dirs[@]}" + + if [ "$user" = "0" ]; then + # this will cause less disk access than `chown -R` + find "${!dirs[@]}" \! -user mysql -exec chown --no-dereference mysql '{}' + + fi +} + +# initializes the database directory +docker_init_database_dir() { + mysql_note "Initializing database files" + "$@" --initialize-insecure --default-time-zone=SYSTEM + mysql_note "Database files initialized" +} + +# Loads various settings that are used elsewhere in the script +# This should be called after mysql_check_config, but before any other functions +docker_setup_env() { + # Get config + declare -g DATADIR SOCKET + DATADIR="$(mysql_get_config 'datadir' "$@")" + SOCKET="$(mysql_get_config 'socket' "$@")" + + # Initialize values that might be stored in a file + file_env 'MYSQL_ROOT_HOST' '%' + file_env 'MYSQL_DATABASE' + file_env 'MYSQL_USER' + file_env 'MYSQL_PASSWORD' + file_env 'MYSQL_ROOT_PASSWORD' + + declare -g DATABASE_ALREADY_EXISTS + if [ -d "$DATADIR/mysql" ]; then + DATABASE_ALREADY_EXISTS='true' + fi +} + +# Execute sql script, passed via stdin +# usage: docker_process_sql [--dont-use-mysql-root-password] [mysql-cli-args] +# ie: docker_process_sql --database=mydb <<<'INSERT ...' +# ie: docker_process_sql --dont-use-mysql-root-password --database=mydb /dev/null + + docker_init_database_dir "$@" + + mysql_note "Starting temporary server" + docker_temp_server_start "$@" + mysql_note "Temporary server started." + + mysql_socket_fix + docker_setup_db + docker_process_init_files /docker-entrypoint-initdb.d/* + + mysql_expire_root_user + + mysql_note "Stopping temporary server" + docker_temp_server_stop + mysql_note "Temporary server stopped" + + echo + mysql_note "MySQL init process done. Ready for start up." + echo + else + mysql_socket_fix + fi + fi + exec "$@" +} + +# If we are sourced from elsewhere, don't perform any further actions +if ! _is_sourced; then + _main "$@" +fi diff --git a/scripts/backup_databases.sh b/scripts/backup_databases.sh new file mode 100644 index 0000000..5737fc7 --- /dev/null +++ b/scripts/backup_databases.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Database credentials +HOST="10.5.48.47" +PORT="3307" +USER="root" +PASSWORD="xM9A4!exJbQNhqJ" + +# Directories +BACKUP_DIR="/app/simfony_deployments/mysql/mysql_dump_bkp/backup_dump" +LOG_FILE="/app/simfony_deployments/mysql/mysql_dump_bkp/backup.log" + +# Timestamp format +CURRENT_HOUR=$(date +"%Y%m%d%H") +CURRENT_TIME=$(date +"%Y%m%d%H%M") + +# Databases to back up +DATABASES=("ipam" "mobility" "notification" "provisioning" "sso_db" "web_db" "simfony_web_db") + +# Create backup directory if it doesn't exist +mkdir -p $BACKUP_DIR + +# Backup databases +for DB in "${DATABASES[@]}"; do + FILENAME="$BACKUP_DIR/${DB}_$CURRENT_TIME.sql" + mysqldump -h $HOST -P $PORT -u $USER -p$PASSWORD $DB > $FILENAME 2>> $LOG_FILE + + if [ $? -eq 0 ]; then + if [[ "${FILENAME}" != *"${CURRENT_HOUR}.sql" ]]; then + gzip $FILENAME + if [ $? -eq 0 ]; then + echo "[$(date)] Backup and compression successful for database '$DB'. File: $FILENAME.gz" >> $LOG_FILE + else + echo "[$(date)] Backup successful but compression failed for database '$DB'. File: $FILENAME" >> $LOG_FILE + fi + else + echo "[$(date)] Backup successful for database '$DB'. File: $FILENAME" >> $LOG_FILE + fi + else + echo "[$(date)] Backup failed for database '$DB'." >> $LOG_FILE + fi +done + diff --git a/scripts/my.cnf b/scripts/my.cnf new file mode 100644 index 0000000..d8ab9de --- /dev/null +++ b/scripts/my.cnf @@ -0,0 +1,90 @@ +# For advice on how to change settings please see +# http://dev.mysql.com/doc/refman/8.0/en/server-configuration-defaults.html + +[mysqldump] +set-gtid-purged=OFF + +[MYSQLD] +user=mysql +server-id=2000 +bind-address=0.0.0.0 +datadir=/var/lib/mysql +character-set-server = utf8mb4 +collation-server = utf8mb4_unicode_ci +socket=/var/lib/mysql/mysql.sock +pid_file=/var/run/mysqld/mysqld.pid +port=3306 +#log_error=/var/log/mysqld.log +#log-error-verbosity=1 +# log_output = FILE +lower_case_table_names=1 + +#Slow logging +slow_query_log_file=/var/log/mysql-slow.log +long_query_time=2 +slow_query_log=OFF +log_queries_not_using_indexes=OFF + +### INNODB OPTIONS +innodb_buffer_pool_size=43G +# innodb_flush_log_at_trx_commit = 2 +innodb_file_per_table=1 +## You may want to tune the below depending on number of cores and disk sub +innodb_read_io_threads=4 +innodb_write_io_threads=4 +innodb_doublewrite=1 +innodb_log_file_size=1024M +innodb_log_buffer_size=96M +innodb_buffer_pool_instances=8 +innodb_log_files_in_group=2 +innodb_thread_concurrency=0 +# innodb_file_format = barracuda +# innodb_flush_method = O_DIRECT +innodb_rollback_on_timeout=ON +# innodb_locks_unsafe_for_binlog = 1 +innodb_autoinc_lock_mode=2 +## avoid statistics update when doing e.g show tables +innodb_stats_on_metadata=0 +innodb_io_capacity = 20000 +default_storage_engine=innodb + +thread_stack = 1920K + +# OTHER THINGS, BUFFERS ETC +key_buffer_size = 200M +tmp_table_size = 64M +max_heap_table_size = 64M +max_allowed_packet = 512M +skip_name_resolve +memlock=0 +sysdate_is_now=1 +max_connections=10000 +max_user_connections=20000 +thread_cache_size=40 +table_open_cache=1024 + +performance_schema = ON +performance-schema-max-mutex-classes = 0 +performance-schema-max-mutex-instances = 0 + +log_bin=binlog +relay_log=relay-bin +expire_logs_days=7 +read_only=OFF +log_bin_trust_function_creators = 1; + +#validate_password.policy = LOW +#validate_password.special_char_count = 0 +#validate_password.check_user_name = OFF +default-authentication-plugin = caching_sha2_password +#validate_password.mixed_case_count = 0 + +#validate_password.length =6 + +#validate_password.number_count = 0 + +binlog_transaction_dependency_tracking = WRITESET +enforce_gtid_consistency = ON +gtid_mode = ON + + diff --git a/scripts/old_qa b/scripts/old_qa new file mode 100644 index 0000000..fcaacac --- /dev/null +++ b/scripts/old_qa @@ -0,0 +1,33 @@ +# Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +# +# The MySQL Server configuration file. +# +# For explanations see +# http://dev.mysql.com/doc/mysql/en/server-system-variables.html + +[mysqld] +pid-file = /var/run/mysqld/mysqld.pid +socket = /var/run/mysqld/mysqld.sock +datadir = /var/lib/mysql +max_connections = 10000 +lower_case_table_names = 1 +secure-file-priv= NULL +# Disabling symbolic-links is recommended to prevent assorted security risks +symbolic-links=0 + +# Custom config should go here +!includedir /etc/mysql/conf.d/ diff --git a/scripts/orna_conf b/scripts/orna_conf new file mode 100644 index 0000000..c8c4665 --- /dev/null +++ b/scripts/orna_conf @@ -0,0 +1,90 @@ +# For advice on how to change settings please see +# http://dev.mysql.com/doc/refman/8.0/en/server-configuration-defaults.html + +[mysqldump] +set-gtid-purged=OFF + +[MYSQLD] +user=mysql +server-id=2000 +bind-address=0.0.0.0 +datadir=/var/lib/mysql +character-set-server = utf8mb4 +collation-server = utf8mb4_unicode_ci +socket=/var/lib/mysql/mysql.sock +pid_file=/var/run/mysqld/mysqld.pid +port=3306 +log_error=/var/log/mysqld.log +log-error-verbosity=1 +# log_output = FILE +lower_case_table_names=1 + +#Slow logging +slow_query_log_file=/var/log/mysql-slow.log +long_query_time=2 +slow_query_log=OFF +log_queries_not_using_indexes=OFF + +### INNODB OPTIONS +innodb_buffer_pool_size=43G +# innodb_flush_log_at_trx_commit = 2 +innodb_file_per_table=1 +## You may want to tune the below depending on number of cores and disk sub +innodb_read_io_threads=4 +innodb_write_io_threads=4 +innodb_doublewrite=1 +innodb_log_file_size=1024M +innodb_log_buffer_size=96M +innodb_buffer_pool_instances=8 +innodb_log_files_in_group=2 +innodb_thread_concurrency=0 +# innodb_file_format = barracuda +# innodb_flush_method = O_DIRECT +innodb_rollback_on_timeout=ON +# innodb_locks_unsafe_for_binlog = 1 +innodb_autoinc_lock_mode=2 +## avoid statistics update when doing e.g show tables +innodb_stats_on_metadata=0 +innodb_io_capacity = 20000 +default_storage_engine=innodb + +thread_stack = 1920K + +# OTHER THINGS, BUFFERS ETC +key_buffer_size = 200M +tmp_table_size = 64M +max_heap_table_size = 64M +max_allowed_packet = 512M +skip_name_resolve +memlock=0 +sysdate_is_now=1 +max_connections=10000 +max_user_connections=20000 +thread_cache_size=40 +table_open_cache=1024 + +performance_schema = ON +performance-schema-max-mutex-classes = 0 +performance-schema-max-mutex-instances = 0 + +log_bin=binlog +relay_log=relay-bin +expire_logs_days=7 +read_only=OFF +log_bin_trust_function_creators = 1; + +validate_password.policy = LOW +validate_password.special_char_count = 0 +validate_password.check_user_name = OFF +default-authentication-plugin = caching_sha2_password +validate_password.mixed_case_count = 0 + +validate_password.length =6 + +validate_password.number_count = 0 + +binlog_transaction_dependency_tracking = WRITESET +enforce_gtid_consistency = ON +gtid_mode = ON + +