您现在的位置是：首页 >技术杂谈 >项目部署 Java Mysql ES Redis网站首页技术杂谈

数据库及中间件搭建

MySQL

创建应用文件夹

mkdir /application/tools
cd /application/tools/

下载安装包/或者doc文件夹下有tar包

wget https://dev.mysql.com/get/Downloads/MySQL-5.7/mysql-5.7.23-linux-glibc2.12-x86_64.tar.gz

下载依赖环境

yum install  -y  bison-devel ncurses-devel libaio-devel gcc gcc-c++ automake autoconf numactl

解压MySQL5.7安装包

tar xf mysql-5.7.23-linux-glibc2.12-x86_64.tar.gz -C /application
mv /application/mysql-5.7.23-linux-glibc2.12-x86_64  /application/mysql

创建管理用户&&授权

useradd -s /sbin/nologin -M mysql
chown -R mysql.mysql /application/mysql/

创建目录和文件并且分配属主和属组

mkdir -p /data/mysql/{data,logs}
touch /data/mysql/logs/{mysql-error.log,mysql-slow.log}
chown -R mysql.mysql /data/mysql
mkdir /var/run/mysql
chown -R mysql.mysql /var/run/mysql

初始化数据库

/application/mysql/bin/mysqld --initialize-insecure --basedir=/application/mysql/ --datadir=/data/mysql/data --user=mysql

复制脚本及PATH变量赋值

cp /application/mysql/support-files/mysql.server /etc/init.d/mysqld
chmod 700 /etc/init.d/mysqld
echo 'PATH=$PATH:/application/mysql/bin' >> /etc/profile
source /etc/profile

修改脚本启动路径

sed -i 's#/usr/local/mysql#/application/mysql#g' /application/mysql/bin/mysqld_safe /etc/init.d/mysqld

替换 /etc/my.cnf

[mysql]

# CLIENT #
port                           = 3306
socket                         = /tmp/mysql.sock

[mysqld]

# GENERAL #
user                           = mysql
default-storage-engine         = InnoDB
socket                         = /tmp/mysql.sock
pid-file                       = /var/run/mysql/mysql.pid
basedir                        = /application/mysql
server-id                      = 1
port                           = 3306
default-time-zone              = '+08:00'

# MyISAM #
key-buffer-size                = 32M
myisam-recover-options         = FORCE,BACKUP

# SAFETY #
max-allowed-packet             = 16M
max-connect-errors             = 1000000
skip-name-resolve
sql-mode                       = ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,NO_ZERO_DATE,NO_ZERO_IN_DATE,NO_AUTO_VALUE_ON_ZERO
sysdate-is-now                 = 1
innodb                         = FORCE
secure-file-priv               = ''

# DATA STORAGE #
datadir                        = /data/mysql/data/

# BINARY LOGGING #
log-bin                        = /data/mysql/data/mysql-bin
expire-logs-days               = 7
binlog-format                  = mixed
sync-binlog                    = 1
#GTID:
gtid-mode                      = on
enforce-gtid-consistency       = on

# CACHES AND LIMITS #
tmp-table-size                 = 32M
max-heap-table-size            = 32M
query-cache-type               = 0
query-cache-size               = 0
max-connections                = 500
thread-cache-size              = 50
open-files-limit               = 65535
table-definition-cache         = 1024
table-open-cache               = 200

# INNODB #
innodb-flush-method            = O_DIRECT
innodb-log-files-in-group      = 2
innodb-log-file-size           = 256M
innodb-flush-log-at-trx-commit = 1
innodb-file-per-table          = 1
# 根据服务器配置调整
innodb-buffer-pool-size        = 16G

# LOGGING #
log-error                      = /data/mysql/logs/mysql-error.log
log-queries-not-using-indexes  = 1
slow-query-log                 = 1
slow-query-log-file            = /data/mysql/logs/mysql-slow.log

启动MySQL

/etc/init.d/mysqld start

初始化MySQL

mysql_secure_installation

Securing the MySQL server deployment.

Connecting to MySQL using a blank password.
VALIDATE PASSWORD PLUGIN can be used to test passwords
and improve security. It checks the strength of password
and allows the users to set only those passwords which are
secure enough. Would you like to setup VALIDATE PASSWORD plugin?

Press y|Y for Yes, any other key for No: n
Securing the MySQL server deployment.

New password: <==此处输入新密码（如果低于八位数密码，稍后会有系统提示）

Re-enter new password: <==此处重复输入新密码

By default, a MySQL installation has an anonymous user,
allowing anyone to log into MySQL without having to have
a user account created for them. This is intended only for
testing, and to make the installation go a bit smoother.
You should remove them before moving into a production
environment.

Remove anonymous users? (Press y|Y for Yes, any other key for No) : y
Success.


Normally, root should only be allowed to connect from
'localhost'. This ensures that someone cannot guess at
the root password from the network.

Disallow root login remotely? (Press y|Y for Yes, any other key for No) : n

 ... skipping.
By default, MySQL comes with a database named 'test' that
anyone can access. This is also intended only for testing,
and should be removed before moving into a production
environment.


Remove test database and access to it? (Press y|Y for Yes, any other key for No) : y
 - Dropping test database...
Success.

 - Removing privileges on test database...
Success.

Reloading the privilege tables will ensure that all changes
made so far will take effect immediately.

Reload privilege tables now? (Press y|Y for Yes, any other key for No) : y
Success.

All done! 

把 MySQL 添加进启动项

cd /etc/rc.d && chmod +x rc.local && vim rc.local

#开机启动mysql
mkdir -p /var/run/mysql
chown -R mysql.mysql /var/run/mysql
/etc/init.d/mysqld start

关闭防火墙

firewall-cmd --state

如果是running的话，使用以下命令关闭防火墙

systemctl stop firewalld.service #停止firewall
systemctl disable firewalld.service #禁止firewall开机启动

命令参考

重启MySQL

systemctl restart mysqld.service ，或者
/etc/init.d/mysqld [stop|start|restart|reload]

添加sql查询账号，特权为只读

CREATE USER 'bi_user'@'192.168.1.115' IDENTIFIED BY 'U2FsdGVkX1CwhtF6ad2';
GRANT SELECT ON `bi`.* TO 'bi_user'@'192.168.1.115' identified by "U2FsdGVkX1CwhtF6ad2";
GRANT SELECT ON `bi_client_log`.* TO 'bi_user'@'192.168.1.115' identified by "U2FsdGVkX1CwhtF6ad2";
FLUSH PRIVILEGES;

添加web机器访问权限

grant all privileges on *.* to root@'192.168.1.115' identified by "tiantangbi";192.168.1.29
FLUSH PRIVILEGES;
java -jar xxl-job-admin-1.0-SNAPSHOT.jar --spring.profiles.active=tthx

Java

cd /application/tools/

下载或者上传tar

wget https://repo.huaweicloud.com/java/jdk/8u201-b09/jdk-8u201-linux-x64.tar.gz

解压

tar -zxvf jdk-8u201-linux-x64.tar.gz

移动文件夹

mv jdk1.8.0_201 /usr/local/jdk1.8/

配置环境变量

vim /etc/profile

export JAVA_HOME=/usr/local/jdk1.8
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

source /etc/profile

检查是否安装成功

java -version

Zookeeper

cd /application/tools/

下载或者上传tar

wget https://downloads.apache.org/zookeeper/zookeeper-3.6.2/apache-zookeeper-3.6.2-bin.tar.gz

解压

tar -xzvf apache-zookeeper-3.6.2-bin.tar.gz

移动

mv apache-zookeeper-3.6.2-bin /application/zookeeper

修改配置文件

cd /application/zookeeper/conf/
cp zoo_sample.cfg zoo.cfg
vim zoo.cfg

修改日志路径

dataDir=/data/zookeeper

创建目录

mkdir /data/zookeeper
mkdir /data/zookeeper/data
mkdir /data/zookeeper/log

修改环境变量

vim /etc/profile

export ZOOKEEPER_INSTALL=/application/zookeeper/
export PATH=$PATH:$ZOOKEEPER_INSTALL/bin

source /etc/profile

启动ZK

./bin/zkServer.sh start
./bin/zkServer.sh status

Kafka

cd /application/tools/

下载或者上传tar

wget https://downloads.apache.org/kafka/2.7.0/kafka_2.12-2.7.0.tgz

解压

tar -zvxf kafka_2.12-2.7.0.tgz

移动目录

mv kafka_2.12-2.7.0 /application/kafka

修改配置文件

vim kafka/config/server.properties

log.dirs=/data/kafka-logs
zookeeper.connect=localhost:2181 # 根据实际修改
listeners=PLAINTEXT://192.168.1.29:9092
advertised.listeners=PLAINTEXT://192.168.1.29:9092

mkdir /data/kafka-logs

启动kafka

./bin/kafka-server-start.sh -daemon config/server.properties &

相关命令

./bin/kafka-server-stop.sh stop #停止kafka

创建Topic

./kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test1

查看Topic

 ./kafka-topics.sh --list --zookeeper localhost:2181

生产消息

./bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test1

消费topic消息

./bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test1 --from-beginning

查看消费进度

./bin/kafka-consumer-groups.sh --group 消费者组名称 --describe --bootstrap-server 192.168.1.29:9092

./bin/kafka-topics.sh --delete --zookeeper 192.168.1.29:2181 --topic EquipOperating

Redis

yum install gcc-c++

下载或者上传tar

tar xzf redis-5.0.7.tar.gz 
cd redis-5.0.7
make && make install

移动文件夹

mv redis-5.0.7 /application/redis

修改配置文件

vim redis.conf

把文件中的daemonize属性改为yes（表明需要在后台运行）

bind 192.168.1.82(本机ip)

设置开机启动

vi /etc/rc.d/rc.local
/application/redis/src/redis-server /application/redis/redis.conf

然后用命令启动

/application/redis/src/redis-server /application/redis/redis.conf

ElasticSearch

下载或上传安装包

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.3-linux-x86_64.tar.gz

解压

tar -zxvf elasticsearch-7.9.3-linux-x86_64.tar.gz 
mv elasticsearch-7.9.3 /application/elasticsearch

创建用户

groupadd elastic
useradd -g elastic elastic
chown -R elastic.elastic /application/elasticsearch/
chgrp -R elastic /application/elasticsearch

修改配置文件

vim elasticsearch.yml

cluster.name: bi-elasticsearch
node.name: node-1

path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/logs

http.host: 0.0.0.0
network.host: 0.0.0.0
discovery.seed_hosts: ["0.0.0.0", "[::1]"]

cluster.initial_master_nodes: ["node-1"]

xpack.license.self_generated.type: trial
xpack.security.enabled: true

vim jvm.options
-Xms4g
-Xmx4g

创建文件夹

mkdir -p /data/elasticsearch/data
mkdir -p /data/elasticsearch/logs
chown -R elastic:elastic /application/elasticsearch/
chown -R elastic:elastic  /data/elasticsearch/data
chown -R elastic:elastic  /data/elasticsearch/logs

vim /etc/security/limits.conf 

在文件最后，增加如下配置：

* soft nofile 65536
* hard nofile 65536

vim /etc/sysctl.conf

在文件最后添加一行

vm.max_map_count=655360

添加完毕之后，执行命令：

sysctl -p

启动

su elastic
./bin/elasticsearch -d

设置密码

.bin/elasticsearch-setup-passwords interactive

这个时候要设置多个账号密码

如果忘记密码：

curl -XGET "localhost:9200/_cat/indices" -H 'Content-Type: application/json'
curl -XDELETE localhost:9200/.security-7

安装IK分词器

下载 https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.9.3/elasticsearch-analysis-ik-7.9.3.zip

移动zip 包到 /application/elasticsearch/plugins/ik

unzip elasticsearch-analysis-ik-7.9.3.zip

重启即可(kill)

Kibana

下载或上传安装包

wget https://artifacts.elastic.co/downloads/kibana/kibana-7.9.3-linux-x86_64.tar.gz

解压

tar -zxvf kibana-7.9.3-linux-x86_64.tar.gz 
mv kibana-7.9.3-linux-x86_64 /application/kibana

修改配置

server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://192.168.3.95:9200"]
kibana.index: ".kibana"
xpack.monitoring.ui.container.elasticsearch.enabled: true
elasticsearch.username: elastic
elasticsearch.password: aoshu123
i18n.locale: "zh-CN"

授权

chown -R elastic.elastic /application/kibana/

启动

su elastic
nohup ./bin/kibana >/dev/null 2>&1 &

filebeat

wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.9.3-linux-x86_64.tar.gz
tar -zxvf filebeat-7.9.3-linux-x86_64.tar.gz
mv filebeat-7.9.3-linux-x86_64 /application/filebeat
chown -R elastic.elastic /application/filebeat/

修改配置文件

filebeat.inputs:
    # Prepaid 日志
    - input_type: log
      # 这里是容器内的path
      paths:  
          - /data/log/fqcalog/Prepaid/*.log #改对应路径
      # 指定被监控的文件的编码类型，使用plain和utf-8都是可以处理中文日志
      encoding: plain
      # 添加tags
      fields:
        log_topic: "Prepaid"
      # 忽略在指定时间跨度之前修改的任何文件
      ignore_older: 3h
      # 指定的时间内没有被读取，将关闭文件句柄
      close_inactive: 1h
      # 文件被删除时，filebeat关闭文件的处理读取
      close_removed: true
      # 如果文件在磁盘上找不到，将从注册表中清除
      clean_removed: true
      # Filebeat如何积极地抓取新文件进行更新
      backoff: 10s
      # 以多快的频率去prospector指定的目录下面检测文件更新
      scan_frequency: 10s
output:
  kafka:
    hosts: ["192.168.1.29:9092"]
    # message topic selection + partitioning
    topic: '%{[fields.log_topic]}'
    partition.round_robin:
      reachable_only: false

    required_acks: 1
    compression: gzip
    max_message_bytes: 1000000

# 关闭自动添加的字段模板
processors:
- drop_fields:
    fields: ["agent", "ecs", "host", "@metadata"]


logging.level: error

启动

nohup ./filebeat -c filebeat.yml &
su elastic
nohup ./filebeat -e -c filebeat.yml -d "Publish" >/dev/null 2>filebeat.log &
tail -f filebeat.log