目标

实现统一的对外下载访问入口的高可用架构，其中所有的Nginx只做下载用途。

机器	安装说明
192.168.89.101	Libfastcommon，FastDFS
192.168.89.102	Libfastcommon，FastDFS
192.168.89.103	Libfastcommon，FastDFS，nginx，fastdfs-nginx-module
192.168.89.104	Libfastcommon，FastDFS，nginx，fastdfs-nginx-module
192.168.89.105	Libfastcommon，FastDFS，nginx，fastdfs-nginx-module
192.168.89.106	Libfastcommon，FastDFS，nginx，fastdfs-nginx-module

安装部署

安装依赖

[root@centos7-81 local]# yum -y install zlib zlib-devel pcre pcre-devel gcc gcc-c++ openssl openssl-devel libevent libevent-devel perl unzip

因为我本地虚拟机是新安装的，所以都需要安装一下，实际中缺哪项安装哪项即可。

文件准备

文件名	说明
V1.0.7	libfastcommon-1.0.7（FastDFS全节点）
FastDFS_v5.05.tar.gz	（FastDFS全节点）
fastdfs-nginx-module_v1.16.tar.gz	解决组内同步延迟问题（存储节点）
ngx_cache_purge-2.3.tar.gz	清除指定url的缓存

注：①本文档中安装版本都和此表格中版本对应。②安装文件默认存放到主机的/usr/src目录下。

安装步骤

通用文件安装

# 为了安装简便，先将防火墙关闭
[root@centos7-101 ~]# systemctl stop firewalld
# 先将文件上传到101，全节点文件拷贝（102、103、104、105、106）
[root@centos7-101 src]# scp V1.0.7 192.168.89.102:/usr/src
[root@centos7-101 src]# scp FastDFS_v5.05.tar.gz 192.168.89.102:/usr/src

libfastcommon安装（全节点安装）

[root@centos7-101 src]# tar -zxvf V1.0.7 -C /usr/local
[root@centos7-101 src]# cd /usr/local/libfastcommon-1.0.7
[root@centos7-101 libfastcommon-1.0.7]# ./make.sh
[root@centos7-101 libfastcommon-1.0.7]# ./make.sh install

全节点完成安装。

FastDFS_v5.05安装（全节点安装）

[root@centos7-101 src]# tar -zxvf FastDFS_v5.05.tar.gz -C /usr/local
[root@centos7-101 src]# cd /usr/local/FastDFS
[root@centos7-101 FastDFS]# ./make.sh
[root@centos7-101 FastDFS]# ./make.sh install

备注：命令执行完，没有错误日志且/etc/fdfs目录下存在文件即安装成功。

单跟踪节点单组存储节点部分

跟踪器配置

以192.168.89.101节点为例进行如下说明。
（1）配置/etc/fdfs/tracker.conf，先拷贝一份，然后进行如下配置：

base_path=/fastdfs/tracker
# 配置存储器
store_lookup=0

（2）创建base_path目录

[root@centos7-101 fdfs]#  mkdir -p /fastdfs/tracker

（3）测试服务

# 启动服务
[root@centos7-101 src]# fdfs_trackerd /etc/fdfs/tracker.conf 
[root@centos7-101 src]# ps -ef | grep fdfs
[root@centos7-101 src]# ss -tpln

存储器配置

单组单节点配置

以192.168.89.103节点为例进行如下说明。
（1）配置/etc/fdfs/storage.conf，先拷贝一份（storage.conf.sample），然后进行如下配置。

group_name=group1
base_path=/fastdfs/storage
store_path0=/fastdfs/storage
tracker_server=192.168.89.101:22122

（2）创建base_path目录

[root@centos7-103 fdfs]#  mkdir -p /fastdfs/storage

（3）测试【启动服务】

# 启动存储服务
[root@centos7-103 fdfs]# fdfs_storaged /etc/fdfs/storage.conf
# 查看启动日志
[root@centos7-103 ~]# tail -f /fastdfs/storage/logs/storaged.log
[2020-05-20 13:42:09] INFO - FastDFS v5.05, base_path=/fastdfs/storage, store_path_count=1, subdir_count_per_path=256, group_name=group1, run_by_group=, run_by_user=, connect_timeout=30s, network_timeout=60s, port=23000, bind_addr=, client_bind=1, max_connections=256, accept_threads=1, work_threads=4, disk_rw_separated=1, disk_reader_threads=1, disk_writer_threads=1, buff_size=256KB, heart_beat_interval=30s, stat_report_interval=60s, tracker_server_count=1, sync_wait_msec=50ms, sync_interval=0ms, sync_start_time=00:00, sync_end_time=23:59, write_mark_file_freq=500, allow_ip_count=-1, file_distribute_path_mode=0, file_distribute_rotate_count=100, fsync_after_written_bytes=0, sync_log_buff_interval=10s, sync_binlog_buff_interval=10s, sync_stat_file_interval=300s, thread_stack_size=512 KB, upload_priority=10, if_alias_prefix=, check_file_duplicate=0, file_signature_method=hash, FDHT group count=0, FDHT server count=0, FDHT key_namespace=, FDHT keep_alive=0, HTTP server port=8888, domain name=, use_access_log=0, rotate_access_log=0, access_log_rotate_time=00:00, rotate_error_log=0, error_log_rotate_time=00:00, rotate_access_log_size=0, rotate_error_log_size=0, log_file_keep_days=0, file_sync_skip_invalid_record=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s
[2020-05-20 13:42:09] INFO - file: storage_param_getter.c, line: 191, use_storage_id=0, id_type_in_filename=ip, storage_ip_changed_auto_adjust=1, store_path=0, reserved_storage_space=10.00%, use_trunk_file=0, slot_min_size=256, slot_max_size=16 MB, trunk_file_size=64 MB, trunk_create_file_advance=0, trunk_create_file_time_base=02:00, trunk_create_file_interval=86400, trunk_create_file_space_threshold=20 GB, trunk_init_check_occupying=0, trunk_init_reload_from_binlog=0, trunk_compress_binlog_min_interval=0, store_slave_file_use_link=0
[2020-05-20 13:42:09] INFO - file: storage_func.c, line: 254, tracker_client_ip: 192.168.89.103, my_server_id_str: 192.168.89.103, g_server_id_in_filename: 1733929152
[2020-05-20 13:42:09] INFO - local_host_ip_count: 3,  127.0.0.1  10.0.2.103  192.168.89.103
[2020-05-20 13:42:09] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 192.168.89.101:22122, as a tracker client, my ip is 192.168.89.103
[2020-05-20 13:42:39] INFO - file: tracker_client_thread.c, line: 1235, tracker server 192.168.89.101:22122, set tracker leader: 192.168.89.101:22122
[2020-05-20 13:42:09] INFO - FastDFS v5.05, base_path=/fastdfs/storage, store_path_count=1, subdir_count_per_path=256, group_name=group1, run_by_group=, run_by_user=, connect_timeout=30s, network_timeout=60s, port=23000, bind_addr=, client_bind=1, max_connections=256, accept_threads=1, work_threads=4, disk_rw_separated=1, disk_reader_threads=1, disk_writer_threads=1, buff_size=256KB, heart_beat_interval=30s, stat_report_interval=60s, tracker_server_count=1, sync_wait_msec=50ms, sync_interval=0ms, sync_start_time=00:00, sync_end_time=23:59, write_mark_file_freq=500, allow_ip_count=-1, file_distribute_path_mode=0, file_distribute_rotate_count=100, fsync_after_written_bytes=0, sync_log_buff_interval=10s, sync_binlog_buff_interval=10s, sync_stat_file_interval=300s, thread_stack_size=512 KB, upload_priority=10, if_alias_prefix=, check_file_duplicate=0, file_signature_method=hash, FDHT group count=0, FDHT server count=0, FDHT key_namespace=, FDHT keep_alive=0, HTTP server port=8888, domain name=, use_access_log=0, rotate_access_log=0, access_log_rotate_time=00:00, rotate_error_log=0, error_log_rotate_time=00:00, rotate_access_log_size=0, rotate_error_log_size=0, log_file_keep_days=0, file_sync_skip_invalid_record=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s
[2020-05-20 13:42:09] INFO - file: storage_param_getter.c, line: 191, use_storage_id=0, id_type_in_filename=ip, storage_ip_changed_auto_adjust=1, store_path=0, reserved_storage_space=10.00%, use_trunk_file=0, slot_min_size=256, slot_max_size=16 MB, trunk_file_size=64 MB, trunk_create_file_advance=0, trunk_create_file_time_base=02:00, trunk_create_file_interval=86400, trunk_create_file_space_threshold=20 GB, trunk_init_check_occupying=0, trunk_init_reload_from_binlog=0, trunk_compress_binlog_min_interval=0, store_slave_file_use_link=0
[2020-05-20 13:42:09] INFO - file: storage_func.c, line: 254, tracker_client_ip: 192.168.89.103, my_server_id_str: 192.168.89.103, g_server_id_in_filename: 1733929152
[2020-05-20 13:42:09] INFO - local_host_ip_count: 3,  127.0.0.1  10.0.2.103  192.168.89.103
[2020-05-20 13:42:09] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 192.168.89.101:22122, as a tracker client, my ip is 192.168.89.103
[2020-05-20 13:42:39] INFO - file: tracker_client_thread.c, line: 1235, tracker server 192.168.89.101:22122, set tracker leader: 192.168.89.101:22122

（4）测试【上传照片】

# 修改控制器(192.168.89.101)客户端配置文件
[root@centos7-101 fdfs]# vi client.conf 
# base_path=/fastdfs/tracker
# tracker_server=192.168.89.101:22122
# 上传图片
[root@centos7-101 src]# fdfs_upload_file /etc/fdfs/client.conf /usr/src/t-1.jpg
group1/M00/00/00/wKhZZ17Ew1OAYOtBAAKLWRU1Cxw772.jpg

单组多节点配置

以192.168.89.103和192.168.89.105为例进行说明。
192.168.89.105节点配置按照103进行相同配置即可，配置完storage.conf文件后进行如下操作。
（1）启动服务

# 启动存储服务
[root@centos7-103 fdfs]# fdfs_storaged /etc/fdfs/storage.conf
# 查看启动日志
[root@centos7-103 ~]# tail -f /fastdfs/storage/logs/storaged.log
[2020-05-20 16:05:27] INFO - FastDFS v5.05, base_path=/fastdfs/storage, store_path_count=1, subdir_count_per_path=256, group_name=group1, run_by_group=, run_by_user=, connect_timeout=30s, network_timeout=60s, port=23000, bind_addr=, client_bind=1, max_connections=256, accept_threads=1, work_threads=4, disk_rw_separated=1, disk_reader_threads=1, disk_writer_threads=1, buff_size=256KB, heart_beat_interval=30s, stat_report_interval=60s, tracker_server_count=1, sync_wait_msec=50ms, sync_interval=0ms, sync_start_time=00:00, sync_end_time=23:59, write_mark_file_freq=500, allow_ip_count=-1, file_distribute_path_mode=0, file_distribute_rotate_count=100, fsync_after_written_bytes=0, sync_log_buff_interval=10s, sync_binlog_buff_interval=10s, sync_stat_file_interval=300s, thread_stack_size=512 KB, upload_priority=10, if_alias_prefix=, check_file_duplicate=0, file_signature_method=hash, FDHT group count=0, FDHT server count=0, FDHT key_namespace=, FDHT keep_alive=0, HTTP server port=8888, domain name=, use_access_log=0, rotate_access_log=0, access_log_rotate_time=00:00, rotate_error_log=0, error_log_rotate_time=00:00, rotate_access_log_size=0, rotate_error_log_size=0, log_file_keep_days=0, file_sync_skip_invalid_record=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s
[2020-05-20 16:05:27] INFO - file: storage_param_getter.c, line: 191, use_storage_id=0, id_type_in_filename=ip, storage_ip_changed_auto_adjust=1, store_path=0, reserved_storage_space=10.00%, use_trunk_file=0, slot_min_size=256, slot_max_size=16 MB, trunk_file_size=64 MB, trunk_create_file_advance=0, trunk_create_file_time_base=02:00, trunk_create_file_interval=86400, trunk_create_file_space_threshold=20 GB, trunk_init_check_occupying=0, trunk_init_reload_from_binlog=0, trunk_compress_binlog_min_interval=0, store_slave_file_use_link=0
[2020-05-20 16:05:27] INFO - file: storage_func.c, line: 254, tracker_client_ip: 192.168.89.105, my_server_id_str: 192.168.89.105, g_server_id_in_filename: 1767483584
[2020-05-20 16:05:27] INFO - local_host_ip_count: 3,  127.0.0.1  10.0.2.105  192.168.89.105
[2020-05-20 16:05:27] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 192.168.89.101:22122, as a tracker client, my ip is 192.168.89.105
[2020-05-20 16:05:27] INFO - file: tracker_client_thread.c, line: 1235, tracker server 192.168.89.101:22122, set tracker leader: 192.168.89.101:22122
[2020-05-20 16:05:27] INFO - file: storage_sync.c, line: 2698, successfully connect to storage server 192.168.89.103:23000

服务启动后，在103上的图片会自动同步过来。可以查看日志文件，出现如下（时间间隔）：
[2020-05-20 16:12:04] INFO - file: storage_sync.c, line: 2698, successfully connect to storage server 192.168.89.105:23000
（2）测试

# 上传图片
[root@centos7-101 src]# fdfs_upload_file /etc/fdfs/client.conf /usr/src/t-3.jpg 
group1/M00/00/00/wKhZZ17E5S2AMZf2AAKMU4U081o805.jpg

总结（t1-s1-s2）

由192.168.89.101【tracker-1】、192.168.89.103【group1-1】和192.168.89.105【group1-2】完成了单组多节点的搭建。
上传测试：

[root@centos7-101 fdfs]# fdfs_upload_file /etc/fdfs/client.conf /usr/src/t-2.jpg 
group1/M00/00/00/wKhZZ17EqGCAL9j8AAKZ_83sbhk602.jpg

日志分析：

# 105日志
[root@centos7-105 logs]# tail -f storaged.log 
mkdir data path: FD ...
mkdir data path: FE ...
mkdir data path: FF ...
data path: /fastdfs/storage/data, mkdir sub dir done.
[2020-05-20 11:38:53] INFO - file: storage_param_getter.c, line: 191, use_storage_id=0, id_type_in_filename=ip, storage_ip_changed_auto_adjust=1, store_path=0, reserved_storage_space=10.00%, use_trunk_file=0, slot_min_size=256, slot_max_size=16 MB, trunk_file_size=64 MB, trunk_create_file_advance=0, trunk_create_file_time_base=02:00, trunk_create_file_interval=86400, trunk_create_file_space_threshold=20 GB, trunk_init_check_occupying=0, trunk_init_reload_from_binlog=0, trunk_compress_binlog_min_interval=0, store_slave_file_use_link=0
[2020-05-20 11:38:53] INFO - file: storage_func.c, line: 254, tracker_client_ip: 192.168.89.105, my_server_id_str: 192.168.89.105, g_server_id_in_filename: 1767483584
[2020-05-20 11:38:53] INFO - local_host_ip_count: 3,  127.0.0.1  10.0.2.105  192.168.89.105
[2020-05-20 11:38:53] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 192.168.89.101:22122, as a tracker client, my ip is 192.168.89.105
[2020-05-20 11:38:53] INFO - file: tracker_client_thread.c, line: 1235, tracker server 192.168.89.101:22122, set tracker leader: 192.168.89.101:22122
[2020-05-20 11:38:53] INFO - file: storage_sync.c, line: 2698, successfully connect to storage server 192.168.89.103:23000
# 103日志
[root@centos7-103 logs]# tail -f storaged.log
mkdir data path: FE ...
mkdir data path: FF ...
data path: /fastdfs/storage/data, mkdir sub dir done.
[2020-05-20 11:38:42] INFO - file: storage_param_getter.c, line: 191, use_storage_id=0, id_type_in_filename=ip, storage_ip_changed_auto_adjust=1, store_path=0, reserved_storage_space=10.00%, use_trunk_file=0, slot_min_size=256, slot_max_size=16 MB, trunk_file_size=64 MB, trunk_create_file_advance=0, trunk_create_file_time_base=02:00, trunk_create_file_interval=86400, trunk_create_file_space_threshold=20 GB, trunk_init_check_occupying=0, trunk_init_reload_from_binlog=0, trunk_compress_binlog_min_interval=0, store_slave_file_use_link=0
[2020-05-20 11:38:42] INFO - file: storage_func.c, line: 254, tracker_client_ip: 192.168.89.103, my_server_id_str: 192.168.89.103, g_server_id_in_filename: 1733929152
[2020-05-20 11:38:42] INFO - local_host_ip_count: 3,  127.0.0.1  10.0.2.103  192.168.89.103
[2020-05-20 11:38:42] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 192.168.89.101:22122, as a tracker client, my ip is 192.168.89.103
[2020-05-20 11:39:12] INFO - file: tracker_client_thread.c, line: 1235, tracker server 192.168.89.101:22122, set tracker leader: 192.168.89.101:22122
[2020-05-20 11:39:12] INFO - file: storage_sync.c, line: 2698, successfully connect to storage server 192.168.89.105:23000
[2020-05-20 11:39:42] INFO - file: storage_sync.c, line: 2698, successfully connect to storage server 192.168.89.105:23000

多跟踪节点多组存储节点部分

跟踪器配置

以192.168.89.101和192.168.89.102为例进行说明。安装同“单跟踪节点单组存储节点部分跟踪器配置部分”。

存储器配置

• 【192.168.89.103】和【192.168.89.105】

该两个节点都属于group1，修改/etc/fdfs/storage.conf配置如下：

group_name=group1
base_path=/fastdfs/storage
store_path0=/fastdfs/storage
tracker_server=192.168.89.101:22122
tracker_server=192.168.89.102:22122

• 【192.168.89.104】和【192.168.89.106】

该两个节点都属于group2，修改/etc/fdfs/storage.conf配置如下：

group_name=group2
base_path=/fastdfs/storage
store_path0=/fastdfs/storage
tracker_server=192.168.89.101:22122
tracker_server=192.168.89.102:22122

测试

在192.168.89.102（T2）跟踪器上传一个图片，可以看到跟踪器的轮询效果，如下所示：

[root@centos7-102 src]# fdfs_upload_file /etc/fdfs/client.conf /usr/src/t-3.jpg 
group1/M00/00/00/wKhZZ17E9TmADDyNAAKMU4U081o921.jpg
[root@centos7-102 src]# fdfs_upload_file /etc/fdfs/client.conf /usr/src/t-3.jpg 
group2/M00/00/00/wKhZal7E9UKAeIPjAAKMU4U081o129.jpg
[root@centos7-102 src]# fdfs_upload_file /etc/fdfs/client.conf /usr/src/t-3.jpg 
group1/M00/00/00/wKhZaV7E9UiARRYYAAKMU4U081o553.jpg
[root@centos7-102 src]# fdfs_upload_file /etc/fdfs/client.conf /usr/src/t-3.jpg 
group2/M00/00/00/wKhZaF7E9UqAM212AAKMU4U081o334.jpg

安装小结

以上部分完成了FastDFS的安装，已经实现上传文件。为了实现图片展示的高可用，我们还需配置ngx_fastdfs_module的配置。后面章节进行插件的说明。

nginx部署安装

nginx作用：存储节点（103、104、105、106）做静态资源代理；非存储节点（107、108）做反向代理。

fastdfs-nginx-module模块安装

凡是存储节点都需要安装此插件，此案例中103、104、105、106节点都需要安装。
（1）上传、解压fastdfs-nginx-module_v1.16.tar.gz

[root@centos7-103 src]# tar -zxvf fastdfs-nginx-module_v1.16.tar.gz -C /usr/local/

（2）修改config文件

[root@centos7-103 src]# cd /usr/local/fastdfs-nginx-module/src
[root@centos7-103 src]# vi config       # 去掉路径中的local
CORE_INCS="$CORE_INCS /usr/local/include/fastdfs /usr/local/include/fastcommon/"

（3）配置mod_fastdfs.conf文件

[root@centos7-103 src]# cp /usr/local/fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs
[root@centos7-103 src]# vi /etc/fdfs/mod_fastdfs.conf
base_path=/fastdfs/storage
tracker_server=192.168.89.101:22122  # 单跟踪节点
tracker_server=192.168.89.102:22122  # 多跟踪节点
group_name=group1 # 属于哪个分组存储节点，就配置哪个名称
url_have_group_name = true
store_path0=/fastdfs/storage # 当前节点的实际存储位置
group_count = 2  # 0表示单个组，一个以上按照实际分租数量进行配置
[group1]         # 多组才进行如下配置
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/fastdfs/storage
[group2]
group_name=group2
storage_server_port=23000
store_path_count=1
store_path0=/fastdfs/storage

（4）拷贝http.conf、mime.types

[root@centos7-103 src]# cd /usr/local/FastDFS/conf
[root@centos7-103 conf]# cp http.conf mime.types /etc/fdfs

（5）配置软连接

[root@centos7-103 src]# ln -s /fastdfs/storage/data /fastdfs/storage/data/M00

至此Nginx扩展模块已安装完毕。

nginx安装

（1）安装

[root@centos7-103 src]# tar -zxvf nginx-1.16.1.tar.gz -C /usr/local
[root@centos7-103 src]# cd /usr/local/nginx-1.16.1/
[root@centos7-103 nginx-1.16.1]# ./configure --prefix=/usr/local/nginx --add-module=/usr/local/fastdfs-nginx-module/src
[root@centos7-103 nginx-1.16.1]# make && make install

（2）启动服务

[root@centos7-103 nginx-1.16.1]# /usr/local/nginx/sbin/nginx

（3）浏览器测试，输入地址：http://192.168.89.103/
103机器命令行输入：[root@centos7-102 src]# curl http://192.168.89.3
显示“Welcome to nginx”等字样，说明服务启动成功。

nginx.conf配置

location /group1/M00 {ngx_fastdfs_module;}

在浏览器测试：http://192.168.89.103/group1/M00/00/00/wKhZZ17Ew1OAYOtBAAKLWRU1Cxw772.jpg

多存储节点安装配置

（1）mod_fastdfs.conf文件【103、104、105、106】

base_path=/fastdfs/storage
tracker_server=192.168.89.101:22122
tracker_server=192.168.89.102:22122
group_name=group1 # 103 105 为group1，104 106为group2
url_have_group_name = true
store_path0=/fastdfs/storage # 当前节点的实际存储位置
group_count = 2  # 此案例共2组
[group1]         # 多组配置
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/fastdfs/storage
[group2]
group_name=group2
storage_server_port=23000
store_path_count=1
store_path0=/fastdfs/storage

（2）nginx.conf配置

location ~/group([1-2])/M00 {ngx_fastdfs_module;}

其他按照单节点的安装即可（上面三节部分）。

负载测试

（1）启动服务

[root@centos7-101 src]# fdfs_trackerd /etc/fdfs/tracker.conf
[root@centos7-102 src]# fdfs_trackerd /etc/fdfs/tracker.conf
[root@centos7-103 src]# fdfs_storaged /etc/fdfs/storage.conf
[root@centos7-104 src]# fdfs_storaged /etc/fdfs/storage.conf
[root@centos7-105 src]# fdfs_storaged /etc/fdfs/storage.conf
[root@centos7-106 src]# fdfs_storaged /etc/fdfs/storage.conf

（2）日志分析

[root@centos7-101 src]# tail -f /fastdfs/tracker/logs/trackerd.log 
[2020-05-22 10:07:42] INFO - FastDFS v5.05, base_path=/fastdfs/tracker, run_by_group=, run_by_user=, connect_timeout=30s, network_timeout=60s, port=22122, bind_addr=, max_connections=256, accept_threads=1, work_threads=4, store_lookup=0, store_group=, store_server=0, store_path=0, reserved_storage_space=10.00%, download_server=0, allow_ip_count=-1, sync_log_buff_interval=10s, check_active_interval=120s, thread_stack_size=64 KB, storage_ip_changed_auto_adjust=1, storage_sync_file_max_delay=86400s, storage_sync_file_max_time=300s, use_trunk_file=0, slot_min_size=256, slot_max_size=16 MB, trunk_file_size=64 MB, trunk_create_file_advance=0, trunk_create_file_time_base=02:00, trunk_create_file_interval=86400, trunk_create_file_space_threshold=20 GB, trunk_init_check_occupying=0, trunk_init_reload_from_binlog=0, trunk_compress_binlog_min_interval=0, use_storage_id=0, id_type_in_filename=ip, storage_id_count=0, rotate_error_log=0, error_log_rotate_time=00:00, rotate_error_log_size=0, log_file_keep_days=0, store_slave_file_use_link=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s
[2020-05-22 10:07:42] INFO - local_host_ip_count: 3,  127.0.0.1  10.0.2.101  192.168.89.101
[2020-05-22 10:08:08] INFO - file: tracker_relationship.c, line: 383, selecting leader...
[2020-05-22 10:08:09] INFO - file: tracker_service.c, line: 969, the tracker leader is 192.168.89.102:22122
[root@centos7-102 src]# tail -f /fastdfs/tracker/logs/trackerd.log 
[2020-05-22 10:07:51] INFO - FastDFS v5.05, base_path=/fastdfs/tracker, run_by_group=, run_by_user=, connect_timeout=30s, network_timeout=60s, port=22122, bind_addr=, max_connections=256, accept_threads=1, work_threads=4, store_lookup=0, store_group=, store_server=0, store_path=0, reserved_storage_space=10.00%, download_server=0, allow_ip_count=-1, sync_log_buff_interval=10s, check_active_interval=120s, thread_stack_size=64 KB, storage_ip_changed_auto_adjust=1, storage_sync_file_max_delay=86400s, storage_sync_file_max_time=300s, use_trunk_file=0, slot_min_size=256, slot_max_size=16 MB, trunk_file_size=64 MB, trunk_create_file_advance=0, trunk_create_file_time_base=02:00, trunk_create_file_interval=86400, trunk_create_file_space_threshold=20 GB, trunk_init_check_occupying=0, trunk_init_reload_from_binlog=0, trunk_compress_binlog_min_interval=0, use_storage_id=0, id_type_in_filename=ip, storage_id_count=0, rotate_error_log=0, error_log_rotate_time=00:00, rotate_error_log_size=0, log_file_keep_days=0, store_slave_file_use_link=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s
[2020-05-22 10:07:51] INFO - local_host_ip_count: 3,  127.0.0.1  10.0.2.102  192.168.89.102
[2020-05-22 10:08:10] INFO - file: tracker_relationship.c, line: 383, selecting leader...
[2020-05-22 10:08:10] INFO - file: tracker_relationship.c, line: 401, I am the new tracker leader 192.168.89.102:22122
[root@centos7-103 src]# tail -f /fastdfs/storage/logs/storaged.log 
[2020-05-22 10:08:33] INFO - FastDFS v5.05, base_path=/fastdfs/storage, store_path_count=1, subdir_count_per_path=256, group_name=group1, run_by_group=, run_by_user=, connect_timeout=30s, network_timeout=60s, port=23000, bind_addr=, client_bind=1, max_connections=256, accept_threads=1, work_threads=4, disk_rw_separated=1, disk_reader_threads=1, disk_writer_threads=1, buff_size=256KB, heart_beat_interval=30s, stat_report_interval=60s, tracker_server_count=2, sync_wait_msec=50ms, sync_interval=0ms, sync_start_time=00:00, sync_end_time=23:59, write_mark_file_freq=500, allow_ip_count=-1, file_distribute_path_mode=0, file_distribute_rotate_count=100, fsync_after_written_bytes=0, sync_log_buff_interval=10s, sync_binlog_buff_interval=10s, sync_stat_file_interval=300s, thread_stack_size=512 KB, upload_priority=10, if_alias_prefix=, check_file_duplicate=0, file_signature_method=hash, FDHT group count=0, FDHT server count=0, FDHT key_namespace=, FDHT keep_alive=0, HTTP server port=8888, domain name=, use_access_log=0, rotate_access_log=0, access_log_rotate_time=00:00, rotate_error_log=0, error_log_rotate_time=00:00, rotate_access_log_size=0, rotate_error_log_size=0, log_file_keep_days=0, file_sync_skip_invalid_record=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s
[2020-05-22 10:08:33] INFO - file: storage_param_getter.c, line: 191, use_storage_id=0, id_type_in_filename=ip, storage_ip_changed_auto_adjust=1, store_path=0, reserved_storage_space=10.00%, use_trunk_file=0, slot_min_size=256, slot_max_size=16 MB, trunk_file_size=64 MB, trunk_create_file_advance=0, trunk_create_file_time_base=02:00, trunk_create_file_interval=86400, trunk_create_file_space_threshold=20 GB, trunk_init_check_occupying=0, trunk_init_reload_from_binlog=0, trunk_compress_binlog_min_interval=0, store_slave_file_use_link=0
[2020-05-22 10:08:33] INFO - file: storage_func.c, line: 254, tracker_client_ip: 192.168.89.103, my_server_id_str: 192.168.89.103, g_server_id_in_filename: 1733929152
[2020-05-22 10:08:33] INFO - local_host_ip_count: 3,  127.0.0.1  10.0.2.103  192.168.89.103
[2020-05-22 10:08:33] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 192.168.89.102:22122, as a tracker client, my ip is 192.168.89.103
[2020-05-22 10:08:33] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 192.168.89.101:22122, as a tracker client, my ip is 192.168.89.103
[2020-05-22 10:08:33] INFO - file: tracker_client_thread.c, line: 1235, tracker server 192.168.89.102:22122, set tracker leader: 192.168.89.102:22122
[2020-05-22 10:09:04] INFO - file: storage_sync.c, line: 2698, successfully connect to storage server 192.168.89.105:23000
[root@centos7-104 nginx-1.16.1]# tail -f /fastdfs/storage/logs/storaged.log 
[2020-05-22 10:08:09] INFO - FastDFS v5.05, base_path=/fastdfs/storage, store_path_count=1, subdir_count_per_path=256, group_name=group2, run_by_group=, run_by_user=, connect_timeout=30s, network_timeout=60s, port=23000, bind_addr=, client_bind=1, max_connections=256, accept_threads=1, work_threads=4, disk_rw_separated=1, disk_reader_threads=1, disk_writer_threads=1, buff_size=256KB, heart_beat_interval=30s, stat_report_interval=60s, tracker_server_count=2, sync_wait_msec=50ms, sync_interval=0ms, sync_start_time=00:00, sync_end_time=23:59, write_mark_file_freq=500, allow_ip_count=-1, file_distribute_path_mode=0, file_distribute_rotate_count=100, fsync_after_written_bytes=0, sync_log_buff_interval=10s, sync_binlog_buff_interval=10s, sync_stat_file_interval=300s, thread_stack_size=512 KB, upload_priority=10, if_alias_prefix=, check_file_duplicate=0, file_signature_method=hash, FDHT group count=0, FDHT server count=0, FDHT key_namespace=, FDHT keep_alive=0, HTTP server port=8888, domain name=, use_access_log=0, rotate_access_log=0, access_log_rotate_time=00:00, rotate_error_log=0, error_log_rotate_time=00:00, rotate_access_log_size=0, rotate_error_log_size=0, log_file_keep_days=0, file_sync_skip_invalid_record=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s
[2020-05-22 10:08:09] INFO - file: storage_param_getter.c, line: 191, use_storage_id=0, id_type_in_filename=ip, storage_ip_changed_auto_adjust=1, store_path=0, reserved_storage_space=10.00%, use_trunk_file=0, slot_min_size=256, slot_max_size=16 MB, trunk_file_size=64 MB, trunk_create_file_advance=0, trunk_create_file_time_base=02:00, trunk_create_file_interval=86400, trunk_create_file_space_threshold=20 GB, trunk_init_check_occupying=0, trunk_init_reload_from_binlog=0, trunk_compress_binlog_min_interval=0, store_slave_file_use_link=0
[2020-05-22 10:08:09] INFO - file: storage_func.c, line: 254, tracker_client_ip: 192.168.89.104, my_server_id_str: 192.168.89.104, g_server_id_in_filename: 1750706368
[2020-05-22 10:08:09] INFO - local_host_ip_count: 3,  127.0.0.1  10.0.2.104  192.168.89.104
[2020-05-22 10:08:09] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 192.168.89.102:22122, as a tracker client, my ip is 192.168.89.104
[2020-05-22 10:08:09] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 192.168.89.101:22122, as a tracker client, my ip is 192.168.89.104
[2020-05-22 10:08:39] INFO - file: tracker_client_thread.c, line: 1235, tracker server 192.168.89.102:22122, set tracker leader: 192.168.89.102:22122
[2020-05-22 10:09:10] INFO - file: storage_sync.c, line: 2698, successfully connect to storage server 192.168.89.106:23000
[root@centos7-105 src]# tail -f /fastdfs/storage/logs/storaged.log 
[2020-05-22 10:08:35] INFO - FastDFS v5.05, base_path=/fastdfs/storage, store_path_count=1, subdir_count_per_path=256, group_name=group1, run_by_group=, run_by_user=, connect_timeout=30s, network_timeout=60s, port=23000, bind_addr=, client_bind=1, max_connections=256, accept_threads=1, work_threads=4, disk_rw_separated=1, disk_reader_threads=1, disk_writer_threads=1, buff_size=256KB, heart_beat_interval=30s, stat_report_interval=60s, tracker_server_count=2, sync_wait_msec=50ms, sync_interval=0ms, sync_start_time=00:00, sync_end_time=23:59, write_mark_file_freq=500, allow_ip_count=-1, file_distribute_path_mode=0, file_distribute_rotate_count=100, fsync_after_written_bytes=0, sync_log_buff_interval=10s, sync_binlog_buff_interval=10s, sync_stat_file_interval=300s, thread_stack_size=512 KB, upload_priority=10, if_alias_prefix=, check_file_duplicate=0, file_signature_method=hash, FDHT group count=0, FDHT server count=0, FDHT key_namespace=, FDHT keep_alive=0, HTTP server port=8888, domain name=, use_access_log=0, rotate_access_log=0, access_log_rotate_time=00:00, rotate_error_log=0, error_log_rotate_time=00:00, rotate_access_log_size=0, rotate_error_log_size=0, log_file_keep_days=0, file_sync_skip_invalid_record=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s
[2020-05-22 10:08:35] INFO - file: storage_param_getter.c, line: 191, use_storage_id=0, id_type_in_filename=ip, storage_ip_changed_auto_adjust=1, store_path=0, reserved_storage_space=10.00%, use_trunk_file=0, slot_min_size=256, slot_max_size=16 MB, trunk_file_size=64 MB, trunk_create_file_advance=0, trunk_create_file_time_base=02:00, trunk_create_file_interval=86400, trunk_create_file_space_threshold=20 GB, trunk_init_check_occupying=0, trunk_init_reload_from_binlog=0, trunk_compress_binlog_min_interval=0, store_slave_file_use_link=0
[2020-05-22 10:08:35] INFO - file: storage_func.c, line: 254, tracker_client_ip: 192.168.89.105, my_server_id_str: 192.168.89.105, g_server_id_in_filename: 1767483584
[2020-05-22 10:08:36] INFO - local_host_ip_count: 3,  127.0.0.1  10.0.2.105  192.168.89.105
[2020-05-22 10:08:36] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 192.168.89.101:22122, as a tracker client, my ip is 192.168.89.105
[2020-05-22 10:08:36] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 192.168.89.102:22122, as a tracker client, my ip is 192.168.89.105
[2020-05-22 10:08:36] INFO - file: tracker_client_thread.c, line: 1235, tracker server 192.168.89.102:22122, set tracker leader: 192.168.89.102:22122
[2020-05-22 10:08:36] INFO - file: storage_sync.c, line: 2698, successfully connect to storage server 192.168.89.103:23000
[root@centos7-106 nginx-1.16.1]# tail -f /fastdfs/storage/logs/storaged.log 
[2020-05-22 10:08:40] INFO - FastDFS v5.05, base_path=/fastdfs/storage, store_path_count=1, subdir_count_per_path=256, group_name=group2, run_by_group=, run_by_user=, connect_timeout=30s, network_timeout=60s, port=23000, bind_addr=, client_bind=1, max_connections=256, accept_threads=1, work_threads=4, disk_rw_separated=1, disk_reader_threads=1, disk_writer_threads=1, buff_size=256KB, heart_beat_interval=30s, stat_report_interval=60s, tracker_server_count=2, sync_wait_msec=50ms, sync_interval=0ms, sync_start_time=00:00, sync_end_time=23:59, write_mark_file_freq=500, allow_ip_count=-1, file_distribute_path_mode=0, file_distribute_rotate_count=100, fsync_after_written_bytes=0, sync_log_buff_interval=10s, sync_binlog_buff_interval=10s, sync_stat_file_interval=300s, thread_stack_size=512 KB, upload_priority=10, if_alias_prefix=, check_file_duplicate=0, file_signature_method=hash, FDHT group count=0, FDHT server count=0, FDHT key_namespace=, FDHT keep_alive=0, HTTP server port=8888, domain name=, use_access_log=0, rotate_access_log=0, access_log_rotate_time=00:00, rotate_error_log=0, error_log_rotate_time=00:00, rotate_access_log_size=0, rotate_error_log_size=0, log_file_keep_days=0, file_sync_skip_invalid_record=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s
[2020-05-22 10:08:40] INFO - file: storage_param_getter.c, line: 191, use_storage_id=0, id_type_in_filename=ip, storage_ip_changed_auto_adjust=1, store_path=0, reserved_storage_space=10.00%, use_trunk_file=0, slot_min_size=256, slot_max_size=16 MB, trunk_file_size=64 MB, trunk_create_file_advance=0, trunk_create_file_time_base=02:00, trunk_create_file_interval=86400, trunk_create_file_space_threshold=20 GB, trunk_init_check_occupying=0, trunk_init_reload_from_binlog=0, trunk_compress_binlog_min_interval=0, store_slave_file_use_link=0
[2020-05-22 10:08:40] INFO - file: storage_func.c, line: 254, tracker_client_ip: 192.168.89.106, my_server_id_str: 192.168.89.106, g_server_id_in_filename: 1784260800
[2020-05-22 10:08:40] INFO - local_host_ip_count: 3,  127.0.0.1  10.0.2.106  192.168.89.106
[2020-05-22 10:08:40] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 192.168.89.102:22122, as a tracker client, my ip is 192.168.89.106
[2020-05-22 10:08:40] INFO - file: tracker_client_thread.c, line: 310, successfully connect to tracker server 192.168.89.101:22122, as a tracker client, my ip is 192.168.89.106
[2020-05-22 10:08:40] INFO - file: tracker_client_thread.c, line: 1235, tracker server 192.168.89.102:22122, set tracker leader: 192.168.89.102:22122
[2020-05-22 10:08:40] INFO - file: storage_sync.c, line: 2698, successfully connect to storage server 192.168.89.104:23000

（3）上传测试

[root@centos7-101 fdfs]# vi client.conf
[root@centos7-101 fdfs]# fdfs_upload_file /etc/fdfs/client.conf /usr/src/t-2.jpg 
group1/M00/00/00/wKhZZ17HNmSAFufHAAKZ_83sbhk311.jpg
[root@centos7-101 fdfs]# fdfs_upload_file /etc/fdfs/client.conf /usr/src/t-3.jpg 
group2/M00/00/00/wKhZaF7HNmqABRnyAAKMU4U081o247.jpg

（4）访问测试
http://192.168.89.103:8000/group1/M00/00/00/wKhZZ17HNmSAFufHAAKZ_83sbhk311.jpg
http://192.168.89.104:8000/group1/M00/00/00/wKhZZ17HNmSAFufHAAKZ_83sbhk311.jpg
按照道理104应该访问不到，因为group1是在103、105上存储，可实际结果显示没问题。经过测试：在四个存储节点否正常的情况下，使用103、105访问group2的存储，重定向到104；使用104、106访问group1的存储，重定向到103。日志如下：

[root@centos7-104 ~]# tail -f /usr/local/nginx/logs/error.log
2020/05/22 10:21:29 [error] 5189#0: *1 upstream timed out (110: Connection timed out) while reading upstream, client: 192.168.89.1, server: localhost, request: "GET /group1/M00/00/00/wKhZZ17HNmSAFufHAAKZ_83sbhk311.jpg HTTP/1.1", upstream: "http://192.168.89.103:8000/group1/M00/00/00/wKhZZ17HNmSAFufHAAKZ_83sbhk311.jpg?redirect=1", host: "192.168.89.104:8000"

从日志中可以看出，重定向到了103的服务。

反向代理配置

我们使用192.168.89.107、192.168.89.108俩节点进行反向代理配置。
（1）安装nginx

[root@centos7-107 src]# tar -zxvf /usr/src/nginx-1.16.1.tar.gz -C /usr/local
[root@centos7-107 src]# cd /usr/local/nginx-1.16.1
[root@centos7-107 src]# ./configure --prefix=/usr/local/nginx
[root@centos7-107 src]# cd /usr/local/nginx-1.16.1

（2）/usr/local/nginx/conf/nginx.conf配置

upstream fdfs_group1 {server 192.168.89.103:8000 weight=1 max_fails=2 fail_timeout=30s;server 192.168.89.105:8000 weight=1 max_fails=2 fail_timeout=30s;
}upstream fdfs_group2 {server 192.168.89.104:8000 weight=1 max_fails=2 fail_timeout=30s;
server 192.168.89.106:8000 weight=1 max_fails=2 fail_timeout=30s;
}
location /group1/M00 {
proxy_next_upstream http_502 http_504 error timeout invalid_header;
proxy_pass http://fdfs_group1;
expires 30d;
}
location /group2/M00 {
proxy_next_upstream http_502 http_504 error timeout invalid_header;
proxy_pass http://fdfs_group2;
expires 30d;
}

（3）测试
http://192.168.89.107:8081/group2/M00/00/00/wKhZaF7HNmqABRnyAAKMU4U081o247.jpg
http://192.168.89.108:8081/group2/M00/00/00/wKhZaF7HNmqABRnyAAKMU4U081o247.jpg
nginx.conf参考配置：

server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 300m;proxy_redirect off;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
proxy_buffer_size 16k;
proxy_buffers 4 64k;
proxy_busy_buffers_size 128k;
proxy_temp_file_write_size 128k;proxy_cache_path /fastdfs/cache/nginx/proxy_cache levels=1:2 keys_zone=http-cache:200m max_size=1g inactive=30d;
proxy_temp_path /fastdfs/cache/nginx/proxy_cache/tmp;upstream fdfs_group1 {server 192.168.89.106:8888 weight=1 max_fails=2 fail_timeout=30s;server 192.168.89.108:8888 weight=1 max_fails=2 fail_timeout=30s;
}upstream fdfs_group2 {server 192.168.89.105:8888 weight=1 max_fails=2 fail_timeout=30s;server 192.168.89.107:8888 weight=1 max_fails=2 fail_timeout=30s;
}location /group1/M00 {
proxy_next_upstream http_502 http_504 error timeout invalid_header;
proxy_cache http-cache;
proxy_cache_valid 200 304 12h;
proxy_cache_key $uri$is_args$args;
proxy_pass http://fdfs_group1;
expires 30d;
}
location /group2/M00 {
#proxy_next_upstream http_502 http_504 error timeout invalid_header;
#proxy_cache http-cache;
#proxy_cache_valid 200 304 12h;
#proxy_cache_key $uri$is_args$args;
proxy_pass http://fdfs_group2;
expires 30d;
}
#清除缓存的访问权限
location ~/purge(/.*) {
allow 127.0.0.1;
allow 192.168.89.0/24;
deny all;
proxy_cache_purge http-cache $1$is_args$args;
}

缓存设置

命令：

[root@centos7-81 src]# tar -zxvf ngx_cache_purge-2.3.tar.gz -C /usr/local
[root@centos7-83 nginx-1.16.1]# ./configure --prefix=/usr/local/nginx --add-module=/usr/local/ngx_cache_purge-2.3
[root@centos7-83 nginx-1.16.1]# make && make install
mkdir –p /fastdfs/cache/nginx/proxy_cache
mkdir –p /fastdfs/cache/nginx/proxy_cache/tmp

测试：
http://192.168.89.3:8000/group2/M00/00/00/wKhZBl7CNgaAQuHFAAKZ_83sbhk190.jpg
http://192.168.89.4:8000/group2/M00/00/00/wKhZBl7CNgaAQuHFAAKZ_83sbhk190.jpg

keepalived

我们使用192.168.89.109、192.168.89.110俩节点进行keepalived配置。
（1）安装nginx
（2）安装keepalived

[root@centos7-109 src]# tar -zxvf keepalived-2.0.20.tar.gz
[root@centos7-109 src]# cd keepalived-2.0.20
[root@centos7-109 keepalived-2.0.20]# ./configure --prefix=/usr/local/keepalived
[root@centos7-109 keepalived-2.0.20]# make && make install

完成后，在生成路径和文件如下：
/usr/local/keepalived/etc/keepalived/keepalived.conf
/usr/local/keepalived/etc/sysconfig/keepalived
/usr/local/keepalived/sbin/keepalived
（3）初始化及启动

# keepalived启动脚本变量引用文件，默认路径是/etc/sysconfig
[root@centos7-109 src]# cp /usr/local/keepalived/etc/sysconfig/keepalived /etc/sysconfig
# 将keepalived主程序加入到环境变量
[root@centos7-109 sbin]# cp /usr/local/keepalived/sbin/keepalived /usr/sbin
[root@centos7-109 sbin]# cp /usr/src/keepalived-2.0.20/keepalived/etc/init.d/keepalived /etc/init.d/
# 将配置文件放到默认路径下
[root@centos7-109 etc]# mkdir /etc/keepalived
[root@centos7-109 etc]# cp /usr/local/keepalived/etc/keepalived/keepalived.conf /etc/keepalived/

（4）192.168.89.109节点配置

# vi /etc/keepalived/keepalived.conf
! Configuration File for keepalivedglobal_defs {router_id LVS_109
}vrrp_script chk_nginx {script "/etc/keepalived/check_list"interval 2weight -20
}vrrp_instance VI_1 {state MASTERinterface enp0s8virtual_router_id 51priority 100advert_int 1authentication {auth_type PASSauth_pass 1111
}
# 将track_script块加入instance配置快
track_script {chk_nginx # 执行Nginx监控服务
}virtual_ipaddress {192.168.89.100}
}

（5）192.168.89.110节点配置

# vi /etc/keepalived/keepalived.conf
! Configuration File for keepalivedglobal_defs {router_id LVS_110
}vrrp_script chk_nginx {script "/etc/keepalived/check_list"interval 2weight -20
}vrrp_instance VI_1 {state BACKUPinterface enp0s8virtual_router_id 51priority 90advert_int 1authentication {auth_type PASSauth_pass 1111
}
# 将track_script块加入instance配置快
track_script {chk_nginx # 执行Nginx监控服务
}virtual_ipaddress {192.168.89.100}
}

（6）check_list脚本
用于检测nginx存活状态的脚本，如果nginx不存活，则关闭宕掉nginx主机上的keepalived。
因为在keepalived+nginx架构中，nginx宕机后，用户请求会失败，但是keepalived不会进行切换。

#!/bin/shnginxpid=$(ps -C nginx --no-header|wc -l)
#1.判断Nginx是否存活,如果不存活则尝试启动Nginx
if [ $nginxpid -eq 0 ];thensystemctl start nginxsleep 3#2.等待3秒后再次获取一次Nginx状态nginxpid=$(ps -C nginx --no-header|wc -l) #3.再次进行判断, 如Nginx还不存活则停止Keepalived,让地址进行漂移,并退出脚本  if [ $nginxpid -eq 0 ];thensystemctl stop keepalivedfi
fi

（7）测试
浏览器中输入：http://192.168.89.100/
显示109节点的页面，然后停掉109节点的nginx和keepalived服务，在访问可以看到地址进行了漂移，显示110节点的页面。

nginx+keepalived+fastdfs

在《keepalived》章节我们进行了nginx服务高可用的配置，本章节我们将fastdfs的代理加入进来。使用109，110环境进行配置说明。
（1）nginx配置【192.168.89.109和192.168.89.110】

# vi /usr/local/nginx/conf/nginx.conf
upstream fdfs_group1 {  # 添加上游服务器server 192.168.89.103:8000 weight=1 max_fails=2 fail_timeout=30s;server 192.168.89.105:8000 weight=1 max_fails=2 fail_timeout=30s;}upstream fdfs_group2 {  # 添加上游服务器server 192.168.89.104:8000 weight=1 max_fails=2 fail_timeout=30s;server 192.168.89.106:8000 weight=1 max_fails=2 fail_timeout=30s;
}
server {listen       80;server_name  localhost;…………location /group1/M00 {  # 反向代理proxy_pass http://fdfs_group1;expires 30d;}location /group2/M00 {proxy_pass http://fdfs_group2;expires 30d;}
…………

先修改109的配置，然后同步到110，[root@centos7-109 conf]# scp nginx.conf 192.168.89.110:/usr/local/nginx/conf。
（2）启动服务
101、102启动fastdfs tracker服务。
103、104、105、106启动fastdfs storage和nginx服务。
109、110启动keepalived和nginx服务。

# 101 102 tracker服务启动
[root@centos7-101 ~]# fdfs_trackerd /etc/fdfs/tracker.conf
[root@centos7-102 ~]# fdfs_trackerd /etc/fdfs/tracker.conf
# 103 104 105 105 storage、nginx服务启动
[root@centos7-103 ~]# fdfs_storaged /etc/fdfs/storage.conf
[root@centos7-103 ~]# /usr/local/nginx/sbin/nginx
……
# 109 110 keepalived、nginx服务启动

（3）测试
服务都正常启动后，我们进行测试。
首先在101或者102服务器上传一个图片，如下所示：

# 101 上传图片
[root@centos7-101 src]# fdfs_upload_file /etc/fdfs/client.conf t-1.jpg 
group1/M00/00/00/wKhZZ17OEOCAMRH1AAKLWRU1Cxw036.jpg
[root@centos7-101 src]# fdfs_upload_file /etc/fdfs/client.conf t-2.jpg 
group2/M00/00/00/wKhZaF7OETOAcwfhAAKZ_83sbhk275.jpg

使用浏览器访问如下地址：
http://192.168.89.100/group1/M00/00/00/wKhZZ17OEOCAMRH1AAKLWRU1Cxw036.jpg
http://192.168.89.100/group2/M00/00/00/wKhZaF7OETOAcwfhAAKZ_83sbhk275.jpg

web项目集成

（1）依赖jar：fastdfs_client.jar
（2）初始化客户端核心代码，写到静态块或者构造器中：

# class根文件目录
String classPath = 
new File(ImgUploadService.class.getResource("/").getFile()).getCanonicalPath();// 根据fdfs_client.conf
ClientGlobal.init(classPath + File.separator + configFile);trackerClient = new TrackerClient();trackerServer = trackerClient.getConnection();storageServer = null;storageClient = new StorageClient(trackerServer, storageServer);log.info("init: " + storageClient.hashCode());// 给dfs发送一个消息，否则第一次传输会有异常信息ProtoCommon.activeTest(trackerServer.getSocket());

（3）上传核心代码

TrackerServer trackerServer = trackerClient.getConnection();
StorageClient storageClient = new StorageClient(trackerServer, storageServer);
// results数组中第一个元素为存储组名，第二个元素为图片存储路径。String[] results = storageClient.upload_file(fileContent, suffix, null);

（4）下载核心代码

public byte[] getFileByteBuffer(Efilesummary object) throws Exception {String classPath = new File(UploadService.class.getResource("/").getFile()).getCanonicalPath();ClientGlobal.init(classPath + File.separator + configFile);TrackerClient trackerClient = new TrackerClient();TrackerServer trackerServer = trackerClient.getConnection();StorageServer storageServer = null;StorageClient storageClient = new StorageClient(trackerServer, storageServer);return storageClient.download_file(object.getPath(), object.getRemotefsid());}

（5）删除核心代码

public void deleteFile(Efilesummary object) throws Exception {String classPath = new File(UploadService.class.getResource("/").getFile()).getCanonicalPath();ClientGlobal.init(classPath + File.separator + configFile);TrackerClient trackerClient = new TrackerClient();TrackerServer trackerServer = trackerClient.getConnection();StorageServer storageServer = null;StorageClient storageClient = new StorageClient(trackerServer, storageServer);int i = storageClient.delete_file(object.getPath(), object.getRemotefsid());System.out.println(i);}

（6）demo
fastdfs-demo.rar

FastDFS资料

基本模块

Tracker：跟踪控制器，程序中会使用该服务器的IP地址。
Storage：存储器，组和组之间代表扩容关系，组内之间块与块代表备份关系。

高可用

参考资料

https://blog.51cto.com/7072753/2286604
https://blog.csdn.net/wc1695040842/article/details/89766064

删除路由：
ip route del default via 192.168.89.1 这个命令加到 ~/.bashrc

• keepalived资料：
  https://blog.csdn.net/l1028386804/article/details/72801492
  https://www.jianshu.com/p/a6b5ab36292a

日志分析

存储监控

[root@centos7-104 fdfs]# fdfs_monitor /etc/fdfs/storage.conf
[2020-05-20 17:10:26] DEBUG - base_path=/fastdfs/storage, connect_timeout=30, network_timeout=60, tracker_server_count=2, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0server_count=2, server_index=1tracker server is 192.168.89.102:22122group count: 2Group 1:
group name = group1
disk total space = 8662 MB
disk free space = 7544 MB
trunk free space = 0 MB
storage server count = 2
active server count = 2
storage server port = 23000
storage HTTP port = 8888
store path count = 1
subdir count per path = 256
current write server index = 0
current trunk file id = 0Storage 1:id = 192.168.89.103ip_addr = 192.168.89.103  ACTIVEhttp domain = version = 5.05join time = 2020-05-20 11:38:40up time = 2020-05-20 17:05:32total storage = 8662 MBfree storage = 7544 MBupload priority = 10store_path_count = 1subdir_count_per_path = 256storage_port = 23000storage_http_port = 8888current_write_path = 0source storage id = if_trunk_server = 0connection.alloc_count = 256connection.current_count = 1connection.max_count = 1total_upload_count = 4success_upload_count = 4total_append_count = 0success_append_count = 0total_modify_count = 0success_modify_count = 0total_truncate_count = 0success_truncate_count = 0total_set_meta_count = 0success_set_meta_count = 0total_delete_count = 0success_delete_count = 0total_download_count = 0success_download_count = 0total_get_meta_count = 0success_get_meta_count = 0total_create_link_count = 0success_create_link_count = 0total_delete_link_count = 0success_delete_link_count = 0total_upload_bytes = 670980success_upload_bytes = 670980total_append_bytes = 0success_append_bytes = 0total_modify_bytes = 0success_modify_bytes = 0stotal_download_bytes = 0success_download_bytes = 0total_sync_in_bytes = 0success_sync_in_bytes = 0total_sync_out_bytes = 0success_sync_out_bytes = 0total_file_open_count = 4success_file_open_count = 4total_file_read_count = 0success_file_read_count = 0total_file_write_count = 4success_file_write_count = 4last_heart_beat_time = 2020-05-20 17:10:01last_source_update = 2020-05-20 16:11:23last_sync_update = 1970-01-01 08:00:00last_synced_timestamp = 1970-01-01 08:00:00 Storage 2:id = 192.168.89.105ip_addr = 192.168.89.105  ACTIVEhttp domain = version = 5.05join time = 2020-05-20 11:38:50up time = 2020-05-20 17:06:32total storage = 8662 MBfree storage = 7544 MBupload priority = 10store_path_count = 1subdir_count_per_path = 256storage_port = 23000storage_http_port = 8888current_write_path = 0source storage id = 192.168.89.103if_trunk_server = 0connection.alloc_count = 256connection.current_count = 1connection.max_count = 1total_upload_count = 0success_upload_count = 0total_append_count = 0success_append_count = 0total_modify_count = 0success_modify_count = 0total_truncate_count = 0success_truncate_count = 0total_set_meta_count = 0success_set_meta_count = 0total_delete_count = 0success_delete_count = 0total_download_count = 0success_download_count = 0total_get_meta_count = 0success_get_meta_count = 0total_create_link_count = 0success_create_link_count = 0total_delete_link_count = 0success_delete_link_count = 0total_upload_bytes = 0success_upload_bytes = 0total_append_bytes = 0success_append_bytes = 0total_modify_bytes = 0success_modify_bytes = 0stotal_download_bytes = 0success_download_bytes = 0total_sync_in_bytes = 670980success_sync_in_bytes = 670980total_sync_out_bytes = 0success_sync_out_bytes = 0total_file_open_count = 4success_file_open_count = 4total_file_read_count = 0success_file_read_count = 0total_file_write_count = 4success_file_write_count = 4last_heart_beat_time = 2020-05-20 17:10:03last_source_update = 1970-01-01 08:00:00last_sync_update = 2020-05-20 16:12:03last_synced_timestamp = 1970-01-01 08:00:00 (never synced)Group 2:
group name = group2
disk total space = 8662 MB
disk free space = 7545 MB
trunk free space = 0 MB
storage server count = 2
active server count = 2
storage server port = 23000
storage HTTP port = 8888
store path count = 1
subdir count per path = 256
current write server index = 1
current trunk file id = 0Storage 1:id = 192.168.89.104ip_addr = 192.168.89.104  ACTIVEhttp domain = version = 5.05join time = 2020-05-20 16:43:25up time = 2020-05-20 17:05:54total storage = 8662 MBfree storage = 7545 MBupload priority = 10store_path_count = 1subdir_count_per_path = 256storage_port = 23000storage_http_port = 8888current_write_path = 0source storage id = if_trunk_server = 0connection.alloc_count = 256connection.current_count = 1connection.max_count = 1total_upload_count = 1success_upload_count = 1total_append_count = 0success_append_count = 0total_modify_count = 0success_modify_count = 0total_truncate_count = 0success_truncate_count = 0total_set_meta_count = 0success_set_meta_count = 0total_delete_count = 0success_delete_count = 0total_download_count = 0success_download_count = 0total_get_meta_count = 0success_get_meta_count = 0total_create_link_count = 0success_create_link_count = 0total_delete_link_count = 0success_delete_link_count = 0total_upload_bytes = 166995success_upload_bytes = 166995total_append_bytes = 0success_append_bytes = 0total_modify_bytes = 0success_modify_bytes = 0stotal_download_bytes = 0success_download_bytes = 0total_sync_in_bytes = 0success_sync_in_bytes = 0total_sync_out_bytes = 0success_sync_out_bytes = 0total_file_open_count = 1success_file_open_count = 1total_file_read_count = 0success_file_read_count = 0total_file_write_count = 1success_file_write_count = 1last_heart_beat_time = 2020-05-20 17:10:23last_source_update = 2020-05-20 16:47:08last_sync_update = 1970-01-01 08:00:00last_synced_timestamp = 1970-01-01 08:00:00 Storage 2:id = 192.168.89.106ip_addr = 192.168.89.106  ACTIVEhttp domain = version = 5.05join time = 2020-05-20 16:43:33up time = 2020-05-20 17:07:02total storage = 8662 MBfree storage = 7545 MBupload priority = 10store_path_count = 1subdir_count_per_path = 256storage_port = 23000storage_http_port = 8888current_write_path = 0source storage id = 192.168.89.104if_trunk_server = 0connection.alloc_count = 256connection.current_count = 1connection.max_count = 1total_upload_count = 0success_upload_count = 0total_append_count = 0success_append_count = 0total_modify_count = 0success_modify_count = 0total_truncate_count = 0success_truncate_count = 0total_set_meta_count = 0success_set_meta_count = 0total_delete_count = 0success_delete_count = 0total_download_count = 0success_download_count = 0total_get_meta_count = 0success_get_meta_count = 0total_create_link_count = 0success_create_link_count = 0total_delete_link_count = 0success_delete_link_count = 0total_upload_bytes = 0success_upload_bytes = 0total_append_bytes = 0success_append_bytes = 0total_modify_bytes = 0success_modify_bytes = 0stotal_download_bytes = 0success_download_bytes = 0total_sync_in_bytes = 166995success_sync_in_bytes = 166995total_sync_out_bytes = 0success_sync_out_bytes = 0total_file_open_count = 1success_file_open_count = 1total_file_read_count = 0success_file_read_count = 0total_file_write_count = 1success_file_write_count = 1last_heart_beat_time = 2020-05-20 17:10:01last_source_update = 1970-01-01 08:00:00last_sync_update = 2020-05-20 16:47:17last_synced_timestamp = 2020-05-20 16:47:08 (0s delay)