From 07ed797afb4646f8a8eb1f4edfbb0273d7dc870d Mon Sep 17 00:00:00 2001 From: MagenX Date: Fri, 18 Nov 2022 13:45:52 +0100 Subject: [PATCH 01/13] Update magenx.install.latest.sh --- magenx.install.latest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magenx.install.latest.sh b/magenx.install.latest.sh index 36bb606..05ce223 100644 --- a/magenx.install.latest.sh +++ b/magenx.install.latest.sh @@ -1819,7 +1819,7 @@ su ${MAGENTO_OWNER} -s /bin/bash -c "bin/magento config:set dev/caching/cache_us #su ${MAGENTO_OWNER} -s /bin/bash -c "bin/magento module:disable Magento_TwoFactorAuth Magento_AdobeIms Magento_AdobeImsApi Magento_AdminAdobeIms" su ${MAGENTO_OWNER} -s /bin/bash -c "mkdir -p var/tmp" -su ${MAGENTO_OWNER} -s /bin/bash -c "composer require magento/quality-patches cweagans/composer-patches" +su ${MAGENTO_OWNER} -s /bin/bash -c "composer require magento/quality-patches cweagans/composer-patches -n" su ${MAGENTO_OWNER} -s /bin/bash -c "bin/magento setup:upgrade" su ${MAGENTO_OWNER} -s /bin/bash -c "bin/magento deploy:mode:set production" su ${MAGENTO_OWNER} -s /bin/bash -c "bin/magento cache:flush" From ae4942b94f41ef0c78befa5b7915150891d3a07d Mon Sep 17 00:00:00 2001 From: MagenX Date: Fri, 18 Nov 2022 15:40:52 +0100 Subject: [PATCH 02/13] Update magenx.install.latest.sh --- magenx.install.latest.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/magenx.install.latest.sh b/magenx.install.latest.sh index 05ce223..ad4f0f0 100644 --- a/magenx.install.latest.sh +++ b/magenx.install.latest.sh @@ -28,6 +28,7 @@ COMPOSER_VERSION="2.2" RABBITMQ_VERSION="3.8*" MARIADB_VERSION="10.5.16" ELK_VERSION="7.x" +ELK_STACK="elasticsearch kibana logstash" PROXYSQL_VERSION="2.3.x" VARNISH_VERSION="70" @@ -1007,13 +1008,13 @@ autorefresh=1 type=rpm-md EOF echo - dnf -y install --enablerepo=elasticsearch-${ELK_VERSION} elasticsearch kibana - rpm --quiet -q elasticsearch + dnf -y install --enablerepo=elasticsearch-${ELK_VERSION} ${ELK_STACK} + rpm --quiet -q ${ELK_STACK} else wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - echo "deb https://artifacts.elastic.co/packages/${ELK_VERSION}/apt stable main" > /etc/apt/sources.list.d/elastic-${ELK_VERSION}.list apt update - apt -y install elasticsearch kibana + apt -y install ${ELK_STACK} fi if [ "$?" = 0 ]; then echo @@ -1055,9 +1056,9 @@ echo GREENTXT "ELASTCSEARCH ${ELKVER} INSTALLED - OK" echo if [[ "${OS_DISTRO_KEY}" =~ (redhat|amazon) ]]; then - rpm -qa 'elasticsearch*' | awk -v var="${PKG_INSTALLED}" '{print var,$1}' + rpm -qa '${ELK_STACK}' | awk -v var="${PKG_INSTALLED}" '{print var,$1}' else - apt -qq list --installed elasticsearch* 2>/dev/null | awk -v var="${PKG_INSTALLED}" '{print var,$0}' + apt -qq list --installed ${ELK_STACK} 2>/dev/null | awk -v var="${PKG_INSTALLED}" '{print var,$0}' fi else echo @@ -1072,9 +1073,9 @@ echo echo ## keep versions for critical services to avoid disruption if [[ "${OS_DISTRO_KEY}" =~ (redhat|amazon) ]]; then - dnf versionlock add elasticsearch kibana erlang rabbitmq-server + dnf versionlock add ${ELK_STACK} erlang rabbitmq-server else - apt-mark hold elasticsearch kibana erlang rabbitmq-server + apt-mark hold ${ELK_STACK} erlang rabbitmq-server fi echo echo From 951c79db017f615c2f48845a47edb5e083b427f2 Mon Sep 17 00:00:00 2001 From: MagenX Date: Sat, 19 Nov 2022 10:41:19 +0100 Subject: [PATCH 03/13] Update magenx.install.latest.sh --- magenx.install.latest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/magenx.install.latest.sh b/magenx.install.latest.sh index ad4f0f0..4a9922f 100644 --- a/magenx.install.latest.sh +++ b/magenx.install.latest.sh @@ -1053,7 +1053,7 @@ rm -rf /tmp/elasticsearch echo echo -GREENTXT "ELASTCSEARCH ${ELKVER} INSTALLED - OK" +GREENTXT "ELASTICSEARCH ${ELK_VERSION} INSTALLED - OK" echo if [[ "${OS_DISTRO_KEY}" =~ (redhat|amazon) ]]; then rpm -qa '${ELK_STACK}' | awk -v var="${PKG_INSTALLED}" '{print var,$1}' @@ -1062,7 +1062,7 @@ echo fi else echo -REDTXT "ELASTCSEARCH INSTALLATION ERROR" +REDTXT "ELASTICSEARCH INSTALLATION ERROR" exit 1 fi else From 8c5255c1dbd9ddc586b08ced001c23e4df3d600c Mon Sep 17 00:00:00 2001 From: MagenX Date: Sat, 19 Nov 2022 11:00:51 +0100 Subject: [PATCH 04/13] Update magenx.install.latest.sh --- magenx.install.latest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/magenx.install.latest.sh b/magenx.install.latest.sh index 4a9922f..a7e0579 100644 --- a/magenx.install.latest.sh +++ b/magenx.install.latest.sh @@ -1820,6 +1820,7 @@ su ${MAGENTO_OWNER} -s /bin/bash -c "bin/magento config:set dev/caching/cache_us #su ${MAGENTO_OWNER} -s /bin/bash -c "bin/magento module:disable Magento_TwoFactorAuth Magento_AdobeIms Magento_AdobeImsApi Magento_AdminAdobeIms" su ${MAGENTO_OWNER} -s /bin/bash -c "mkdir -p var/tmp" +su ${MAGENTO_OWNER} -s /bin/bash -c "composer config --no-plugins allow-plugins.cweagans/composer-patches true" su ${MAGENTO_OWNER} -s /bin/bash -c "composer require magento/quality-patches cweagans/composer-patches -n" su ${MAGENTO_OWNER} -s /bin/bash -c "bin/magento setup:upgrade" su ${MAGENTO_OWNER} -s /bin/bash -c "bin/magento deploy:mode:set production" From 28a9f488166e8d1bab0d23e06988831113c2cd92 Mon Sep 17 00:00:00 2001 From: MagenX Date: Mon, 21 Nov 2022 13:07:50 +0100 Subject: [PATCH 05/13] Update magenx.install.latest.sh --- magenx.install.latest.sh | 85 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 5 deletions(-) diff --git a/magenx.install.latest.sh b/magenx.install.latest.sh index a7e0579..143df3d 100644 --- a/magenx.install.latest.sh +++ b/magenx.install.latest.sh @@ -39,12 +39,12 @@ REMI_RPM_REPO="http://rpms.famillecollet.com/enterprise/remi-release-8.rpm" # WebStack Packages .deb EXTRA_PACKAGES_DEB="curl jq gnupg2 auditd apt-transport-https apt-show-versions ca-certificates lsb-release make autoconf snapd automake libtool uuid-runtime \ perl openssl unzip recode ed e2fsprogs screen inotify-tools iptables smartmontools clamav mlocate vim wget sudo bc apache2-utils \ -logrotate git python3-pip python3-dateutil python3-dev netcat patch ipset postfix strace rsyslog geoipupdate moreutils lsof xinetd jpegoptim sysstat acl attr iotop expect webp imagemagick snmp" +logrotate git python3-pip python3-dateutil python3-dev netcat patch ipset postfix strace rsyslog geoipupdate moreutils lsof xinetd jpegoptim sysstat acl attr iotop expect imagemagick snmp" PERL_MODULES_DEB="liblwp-protocol-https-perl libdbi-perl libconfig-inifiles-perl libdbd-mysql-perl libterm-readkey-perl" PHP_PACKAGES_DEB=(cli fpm common mysql zip lz4 gd mbstring curl xml bcmath intl ldap soap oauth apcu) # WebStack Packages .rpm -EXTRA_PACKAGES_RPM="autoconf snapd jq automake dejavu-fonts-common dejavu-sans-fonts libtidy libpcap libwebp gettext-devel recode gflags tbb ed lz4 libyaml libdwarf \ +EXTRA_PACKAGES_RPM="autoconf snapd jq automake dejavu-fonts-common dejavu-sans-fonts libtidy libpcap gettext-devel recode gflags tbb ed lz4 libyaml libdwarf \ bind-utils e2fsprogs svn screen gcc iptraf inotify-tools iptables smartmontools net-tools mlocate unzip vim wget curl sudo bc mailx clamav-filesystem clamav-server \ clamav-update clamav-milter-systemd clamav-data clamav-server-systemd clamav-scanner-systemd clamav clamav-milter clamav-lib logrotate git patch ipset strace rsyslog \ ncurses-devel GeoIP GeoIP-devel s3cmd geoipupdate openssl-devel ImageMagick libjpeg-turbo-utils pngcrush jpegoptim moreutils lsof net-snmp net-snmp-utils xinetd \ @@ -1017,7 +1017,8 @@ echo apt -y install ${ELK_STACK} fi if [ "$?" = 0 ]; then - echo +echo +## elasticsearch settings echo "discovery.type: single-node" >> /etc/elasticsearch/elasticsearch.yml echo "xpack.security.enabled: true" >> /etc/elasticsearch/elasticsearch.yml sed -i "s/.*cluster.name.*/cluster.name: magento/" /etc/elasticsearch/elasticsearch.yml @@ -1049,6 +1050,69 @@ REMOTE_MONITORING_USER_PASSWORD="$(awk '/PASSWORD remote_monitoring_user/ { prin ELASTIC_PASSWORD="$(awk '/PASSWORD elastic/ { print $4 }' /tmp/elasticsearch)" END +include_config ${MAGENX_CONFIG_PATH}/elasticsearch + +## kibana settings +KIBANA_PORT=$(shuf -i 15741-15997 -n 1) +sed -i "s/#server.port:/server.port: ${KIBANA_PORT}/" /etc/kibana/kibana.yml +sed -i "s/#server.host:/server.host: "0.0.0.0"/" /etc/kibana/kibana.yml +sed -i "s/#elasticsearch.username:/elasticsearch.username: "kibana_system"/" /etc/kibana/kibana.yml +sed -i "s/#elasticsearch.password:/elasticsearch.password: "${KIBANA_SYSTEM_PASSWORD}"/" /etc/kibana/kibana.yml + +echo +GREENTXT "CREATE ELK USERS FOR LOGGING AND INDEXER:" +for elk_user in logstash indexer +do + +curl -X POST -u elastic:${ELASTIC_PASSWORD} "http://127.0.0.1:9200/_security/role/magento_${elk_user}" -H 'Content-Type: application/json' -d "$(cat <> ${MAGENX_CONFIG_PATH}/elasticsearch + +done + +curl -X PUT -u elastic:${ELASTIC_PASSWORD} "http://127.0.0.1:9200/_ilm/policy/magento_logstash" -H 'Content-Type: application/json' -d "$(cat <> /etc/sysctl.conf < Date: Mon, 21 Nov 2022 13:13:17 +0100 Subject: [PATCH 06/13] Create logstash.conf --- logstash.conf | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 logstash.conf diff --git a/logstash.conf b/logstash.conf new file mode 100644 index 0000000..1c48524 --- /dev/null +++ b/logstash.conf @@ -0,0 +1,36 @@ +input { + file { + path => "MAGENTO_WEB_ROOT_PATH/var/log/system.log" + codec => multiline { + pattern => "^\[%{TIMESTAMP_ISO8601}\]" + negate => true + what => "previous" + } + } +} + +filter { + grok { + match => { + "message" => "\[%{TIMESTAMP_ISO8601:date}\] %{DATA}.%{LOGLEVEL:severity}: %{GREEDYDATA:message}" + } + } + + date { + match => ["date", "ISO8601"] + timezone => "MAGENTO_TIMEZONE" + target => "@timestamp" + } +} + +output { + if "_grokparsefailure" not in [tags] { + elasticsearch { + hosts => ["http://127.0.0.1:9200"] + index => "MAGENTO_DOMAIN-log-%{+YYYY.MM.dd}" + ilm_policy => "MAGENTO_LOGSTASH_POLICY" + user => "magento_logstash" + password => "MAGENTO_LOGSTASH_PASSWORD" + } + } +} From 62bbfdca68f906945959555506b2242bd4a1238f Mon Sep 17 00:00:00 2001 From: MagenX Date: Mon, 21 Nov 2022 13:14:52 +0100 Subject: [PATCH 07/13] Update logstash.conf --- logstash.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logstash.conf b/logstash.conf index 1c48524..76001a1 100644 --- a/logstash.conf +++ b/logstash.conf @@ -28,9 +28,9 @@ output { elasticsearch { hosts => ["http://127.0.0.1:9200"] index => "MAGENTO_DOMAIN-log-%{+YYYY.MM.dd}" - ilm_policy => "MAGENTO_LOGSTASH_POLICY" + ilm_policy => "MAGENTO_LOGSTASH_POLICY" user => "magento_logstash" - password => "MAGENTO_LOGSTASH_PASSWORD" + password => "MAGENTO_LOGSTASH_PASSWORD" } } } From 3ad593f17e9f69b94d64b288ae78062e7dcefd0c Mon Sep 17 00:00:00 2001 From: MagenX Date: Mon, 21 Nov 2022 13:17:55 +0100 Subject: [PATCH 08/13] Update magenx.install.latest.sh --- magenx.install.latest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/magenx.install.latest.sh b/magenx.install.latest.sh index 143df3d..882e191 100644 --- a/magenx.install.latest.sh +++ b/magenx.install.latest.sh @@ -1114,6 +1114,7 @@ EOF )" rm -rf /tmp/elasticsearch +rm /etc/logstash/logstash-sample.conf echo echo From 297be9f4821804f87b01335a1224494c1232c16c Mon Sep 17 00:00:00 2001 From: MagenX Date: Mon, 21 Nov 2022 14:02:58 +0100 Subject: [PATCH 09/13] Update logstash.conf --- logstash.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash.conf b/logstash.conf index 76001a1..9f621c4 100644 --- a/logstash.conf +++ b/logstash.conf @@ -28,7 +28,7 @@ output { elasticsearch { hosts => ["http://127.0.0.1:9200"] index => "MAGENTO_DOMAIN-log-%{+YYYY.MM.dd}" - ilm_policy => "MAGENTO_LOGSTASH_POLICY" + ilm_policy => "magento_logstash" user => "magento_logstash" password => "MAGENTO_LOGSTASH_PASSWORD" } From 00463cd42d598ca6b94ab47299a91be33a91a415 Mon Sep 17 00:00:00 2001 From: MagenX Date: Mon, 21 Nov 2022 14:11:01 +0100 Subject: [PATCH 10/13] Update magenx.install.latest.sh --- magenx.install.latest.sh | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/magenx.install.latest.sh b/magenx.install.latest.sh index 882e191..66463c3 100644 --- a/magenx.install.latest.sh +++ b/magenx.install.latest.sh @@ -1059,6 +1059,8 @@ sed -i "s/#server.host:/server.host: "0.0.0.0"/" /etc/kibana/kibana.yml sed -i "s/#elasticsearch.username:/elasticsearch.username: "kibana_system"/" /etc/kibana/kibana.yml sed -i "s/#elasticsearch.password:/elasticsearch.password: "${KIBANA_SYSTEM_PASSWORD}"/" /etc/kibana/kibana.yml +YELLOWTXT "KIBANA PORT :${KIBANA_PORT}" + echo GREENTXT "CREATE ELK USERS FOR LOGGING AND INDEXER:" for elk_user in logstash indexer @@ -1076,8 +1078,8 @@ curl -X POST -u elastic:${ELASTIC_PASSWORD} "http://127.0.0.1:9200/_security/rol } EOF )" - -USER_PASSWORD=$(head -c 500 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 15 | head -n 1) +echo +USER_PASSWORD=$(head -c 500 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) curl -X POST -u elastic:${ELASTIC_PASSWORD} "http://127.0.0.1:9200/_security/user/magento_${elk_user}" --header 'Content-Type: application/json' -d "$(cat <> ${MAGENX_CONFIG_PATH}/elasticsearch done - +echo curl -X PUT -u elastic:${ELASTIC_PASSWORD} "http://127.0.0.1:9200/_ilm/policy/magento_logstash" -H 'Content-Type: application/json' -d "$(cat <> /etc/sysctl.conf < ${MAGENX_CONFIG_PATH}/public_html.acl echo From 031e5245d9890ed5e710474ce8ea8a442b547ae5 Mon Sep 17 00:00:00 2001 From: MagenX Date: Mon, 21 Nov 2022 14:43:09 +0100 Subject: [PATCH 11/13] Update magenx.install.latest.sh --- magenx.install.latest.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/magenx.install.latest.sh b/magenx.install.latest.sh index 66463c3..668e958 100644 --- a/magenx.install.latest.sh +++ b/magenx.install.latest.sh @@ -1058,7 +1058,7 @@ sed -i "s/#server.port:/server.port: ${KIBANA_PORT}/" /etc/kibana/kibana.yml sed -i "s/#server.host:/server.host: "0.0.0.0"/" /etc/kibana/kibana.yml sed -i "s/#elasticsearch.username:/elasticsearch.username: "kibana_system"/" /etc/kibana/kibana.yml sed -i "s/#elasticsearch.password:/elasticsearch.password: "${KIBANA_SYSTEM_PASSWORD}"/" /etc/kibana/kibana.yml - +echo YELLOWTXT "KIBANA PORT :${KIBANA_PORT}" echo @@ -1504,8 +1504,8 @@ WHITETXT "---------------------------------------------------------------------- echo ## logstash settings curl -o /etc/logstash/conf.d/magento.conf -s ${MAGENX_MSI_REPO}logstash.conf -sed -i "s/MAGENTO_WEB_ROOT_PATH/${MAGENTO_WEB_ROOT_PATH}/" /etc/logstash/conf.d/magento.conf -sed -i "s/MAGENTO_TIMEZONE/${MAGENTO_TIMEZONE}/" /etc/logstash/conf.d/magento.conf +sed -i "s|MAGENTO_WEB_ROOT_PATH|${MAGENTO_WEB_ROOT_PATH}|" /etc/logstash/conf.d/magento.conf +sed -i "s|MAGENTO_TIMEZONE|${MAGENTO_TIMEZONE}|" /etc/logstash/conf.d/magento.conf sed -i "s/MAGENTO_DOMAIN/${MAGENTO_DOMAIN}/" /etc/logstash/conf.d/magento.conf sed -i "s/MAGENTO_LOGSTASH_PASSWORD/${MAGENTO_LOGSTASH_PASSWORD}/" /etc/logstash/conf.d/magento.conf From 0b255b1838ce0ef3ef829f1ce0d87f9b232f9ab0 Mon Sep 17 00:00:00 2001 From: MagenX Date: Mon, 21 Nov 2022 15:34:08 +0100 Subject: [PATCH 12/13] Update magenx.install.latest.sh --- magenx.install.latest.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/magenx.install.latest.sh b/magenx.install.latest.sh index 668e958..7ac932d 100644 --- a/magenx.install.latest.sh +++ b/magenx.install.latest.sh @@ -1495,7 +1495,7 @@ if [[ "${OS_DISTRO_KEY}" =~ (redhat|amazon) ]]; then PHP_VERSION="$(php -v | head -n 1 | cut -d " " -f 2 | cut -f1-2 -d".")" php_ini="/etc/php/${PHP_VERSION}/fpm/php.ini" php_fpm_pool_path="/etc/php/${PHP_VERSION}/fpm/pool.d/" - php_ini_path_overrides="/etc/php/${PHP_VERSION}/cli/conf.d/" + php_ini_path_overrides="/etc/php/${PHP_VERSION}/{cli,fpm}/conf.d/" fi echo @@ -1567,22 +1567,22 @@ wget -qO /usr/local/bin/mysqltuner ${MYSQL_TUNER} wget -qO /usr/local/bin/mytop ${MYSQL_TOP} if [ "${OS_DISTRO_KEY}" == "redhat" ]; then -cat < /etc/yum.repos.d/proxysql.repo < /etc/yum.repos.d/proxysql.repo < ${php_ini_path_overrides}/zz-${MAGENTO_OWNER}-overrides.ini < Date: Mon, 21 Nov 2022 15:39:15 +0100 Subject: [PATCH 13/13] Update magenx.install.latest.sh --- magenx.install.latest.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/magenx.install.latest.sh b/magenx.install.latest.sh index 7ac932d..9eabf93 100644 --- a/magenx.install.latest.sh +++ b/magenx.install.latest.sh @@ -1054,10 +1054,10 @@ include_config ${MAGENX_CONFIG_PATH}/elasticsearch ## kibana settings KIBANA_PORT=$(shuf -i 15741-15997 -n 1) -sed -i "s/#server.port:/server.port: ${KIBANA_PORT}/" /etc/kibana/kibana.yml -sed -i "s/#server.host:/server.host: "0.0.0.0"/" /etc/kibana/kibana.yml -sed -i "s/#elasticsearch.username:/elasticsearch.username: "kibana_system"/" /etc/kibana/kibana.yml -sed -i "s/#elasticsearch.password:/elasticsearch.password: "${KIBANA_SYSTEM_PASSWORD}"/" /etc/kibana/kibana.yml +sed -i "s/.*#server.port:.*/server.port: ${KIBANA_PORT}/" /etc/kibana/kibana.yml +sed -i "s/.*#server.host:.*/server.host: \"0.0.0.0\"/" /etc/kibana/kibana.yml +sed -i "s/.*#elasticsearch.username:.*/elasticsearch.username: \"kibana_system\"/" /etc/kibana/kibana.yml +sed -i "s/.*#elasticsearch.password:.*/elasticsearch.password: \"${KIBANA_SYSTEM_PASSWORD}\"/" /etc/kibana/kibana.yml echo YELLOWTXT "KIBANA PORT :${KIBANA_PORT}"