Mogilefs

Mogilefs搭建

环境

master1:nginx反代节点master2、3、4：成为mogilefs节点master2:MySQL节点

0、安装依赖环境包

[root@master2 ~]# yum install -y perl-Sys-Syslog perl-Net-Netmask perl-IO-AIO[root@master3 ~]# yum install -y perl-Sys-Syslog perl-Net-Netmask perl-IO-AIO[root@master4 ~]# yum install -y perl-Sys-Syslog perl-Net-Netmask perl-IO-AIOyum install -y perl-core

1、mogilefs组件安装

[root@master2 mogilefs]# lsMogileFS-Server-2.46-2.el6.noarch.rpm            MogileFS-Utils-2.19-1.el6.noarch.rpm        perl-MogileFS-Client-1.14-1.el6.noarch.rpmMogileFS-Server-mogilefsd-2.46-2.el6.noarch.rpm  Perlbal-1.78-1.el6.noarch.rpm               perl-Perlbal-1.78-1.el6.noarch.rpmMogileFS-Server-mogstored-2.46-2.el6.noarch.rpm  perl-Danga-Socket-1.61-1.el6.rf.noarch.rpm[root@master3 mogilefs]# yum install -y *.rpm3、4节点同上

2、master搭建mysql

2.1 MariaDB配置

[root@master2 ~]# vim /etc/my.cnf[mysqld]datadir=/var/lib/mysqlsocket=/var/lib/mysql/mysql.sockinnodb_file_per_table = 1skip_name_resolve = 1[root@master2 ~]# systemctl start mariadb.service设置开机启动mariadb：[root@master2 ~]# systemctl enable mariadb.service

2.2 mysql创建数据库和用户

创建远程连接的root用户：MariaDB [(none)]> GRANT ALL ON *.* TO 'root'@'10.201.106.%' IDENTIFIED BY 'mageedu';创建数据库：MariaDB [(none)]> CREATE DATABASE mogilefs;创建普通账号：MariaDB [(none)]> GRANT ALL ON mogilefs.* TO 'moguser'@'10.201.106.%' IDENTIFIED BY 'mogpass';MariaDB [(none)]> FLUSH PRIVILEGES;

2.3 使用mogdbsetup使数据初始化

[root@master2 ~]# mogdbsetup --dbhost=10.201.106.132 --dbrootpass='magedu' --dbuser='moguser' --dbpass='mogpass'查看mogilefs数据库的表格MariaDB [mogilefs]> SHOW TABLES;+----------------------+| Tables_in_mogilefs   |+----------------------+| checksum             || class                || device               || domain               || file                 || file_on              || file_on_corrupt      || file_to_delete       || file_to_delete2      || file_to_delete_later || file_to_queue        || file_to_replicate    || fsck_log             || host                 || server_settings      || tempfile             || unreachable_fids     |+----------------------+17 rows in set (0.00 sec)

3、配置mogilefs

3.1 master2：tracker配置

[root@master2 mogilefs]# vim mogilefsd.conf # Enable daemon mode to work in background and use syslogdaemonize = 1# Where to store the pid of the daemon (must be the same in the init script)pidfile = /var/run/mogilefsd/mogilefsd.pid# Database connection informationdb_dsn = DBI:mysql:mogilefs:host=10.201.106.132db_user = moguserdb_pass = mogpass# IP:PORT to listen on for mogilefs client requestslisten = 0.0.0.0:7001

3.2

[root@master2 mogilefs]# rpm -qi MogileFS-Server-mogilefsdName        : MogileFS-Server-mogilefsdVersion     : 2.46Release     : 2.el6Architecture: noarchInstall Date: Mon 16 Jan 2017 05:54:44 AM CSTGroup       : Applications/CPANSize        : 535807License     : ArtisticSignature   : (none)Source RPM  : MogileFS-Server-2.46-2.el6.src.rpmBuild Date  : Tue 08 Oct 2013 06:48:24 PM CSTBuild Host  : www.magedu.comRelocations : (not relocatable)Packager    : MageEdu 
    
     Vendor      : Alan Kasindorf 
     
      Summary     : MogileFS-Server-mogilefsd - Mogilefsd and related libraries.Description :[root@master2 mogilefsd]# service mogilefsd start
     
    

4、mogilestorage

4.1 创建数据存储目录

[root@master2 ~]# mkdir -pv /data/mogilefs/dev1mkdir: created directory ‘/data’mkdir: created directory ‘/data/mogilefs’mkdir: created directory ‘/data/mogilefs/dev1’[root@master2 ~]# chown -R mogilefs.mogilefs /data/mogilefs/

4.2 配置文件

[root@master2 ~]# vim /etc/mogilefs/mogstored.conf maxconns = 10000httplisten = 0.0.0.0:7500mgmtlisten = 0.0.0.0:7501docroot = /data/mogilefs/

4.3 启动服务

[root@master2 ~]# rpm -ql MogileFS-Server-mogstored/etc/mogilefs/mogstored.conf/etc/rc.d/init.d/mogstored/usr/bin/mogautomount/usr/bin/mogstored/usr/share/man/man1/mogautomount.1.gz/usr/share/man/man1/mogstored.1.gz/usr/share/perl5/vendor_perl/Mogstored/ChildProcess/usr/share/perl5/vendor_perl/Mogstored/ChildProcess.pm/usr/share/perl5/vendor_perl/Mogstored/ChildProcess/DiskUsage.pm/usr/share/perl5/vendor_perl/Mogstored/ChildProcess/IOStat.pm/usr/share/perl5/vendor_perl/Mogstored/FIDStatter.pm/usr/share/perl5/vendor_perl/Mogstored/HTTPServer/usr/share/perl5/vendor_perl/Mogstored/HTTPServer.pm/usr/share/perl5/vendor_perl/Mogstored/HTTPServer/Apache.pm/usr/share/perl5/vendor_perl/Mogstored/HTTPServer/Lighttpd.pm/usr/share/perl5/vendor_perl/Mogstored/HTTPServer/Nginx.pm/usr/share/perl5/vendor_perl/Mogstored/HTTPServer/None.pm/usr/share/perl5/vendor_perl/Mogstored/HTTPServer/Perlbal.pm/usr/share/perl5/vendor_perl/Mogstored/SideChannelClient.pm/usr/share/perl5/vendor_perl/Mogstored/SideChannelListener.pm/usr/share/perl5/vendor_perl/Mogstored/TaskQueue.pm[root@master2 ~]# vim /etc/rc.d/init.d/mogstored#!/bin/bash# Author: MageEdu 
    
     # mogstored - Startup script for the MogileFS storage## chkconfig: - 86 14# description: MogileFS storage # processname: mogstored# config: /etc/mogilefs/mogstored.conf # pidfile: /var/run/mogilefsd/mogstored.pid# Source function library.. /etc/rc.d/init.d/functions# Path to the apachectl script, server binary, and short-form for messages.pidfile=/var/run/mogilefs/mogstored.pidlockfile=${LOCKFILE-/var/lock/subsys/mogstored}RETVAL=0configfile='/etc/mogilefs/mogstored.conf'prog=$(which mogstored)        echo -n $"Starting mogstored"        su - mogilefs -c "$prog -c $configfile --daemon"  &> /dev/null        RETVAL=$?        echo        echo -n $"Starting mogstored"        su - mogilefs -c "$prog -c $configfile --daemon"  &> /dev/null        RETVAL=$?        [ $RETVAL = 0 ] && success && touch ${lockfile} && echo $(pidof mogstored) > ${pidfile} || failure        echo        return $RETVAL}stop() {        echo -n $"Stopping mogstored"         netstat -nlp|grep "mogstored"|grep -v grep|awk '{print $7}'|awk -F"/" '{print $1}'|xargs kill -9         RETVAL=$?        [ $RETVAL = 0 ] && success && rm -f ${lockfile} ${pidfile} || failure        echo}[root@master2 ~]# systemctl daemon-reload[root@master2 ~]# service mogstored start
    

4.4 master3,4 storage配置

[root@master3 ~]# mkdir -pv /data/mogilefs/dev2mkdir: created directory ‘/data’mkdir: created directory ‘/data/mogilefs’mkdir: created directory ‘/data/mogilefs/dev2’[root@master3 ~]# chown -R mogilefs.mogilefs /data/mogilefs/[root@master4 ~]# mkdir -pv /data/mogilefs/dev3mkdir: created directory ‘/data’mkdir: created directory ‘/data/mogilefs’mkdir: created directory ‘/data/mogilefs/dev3’[root@master4 ~]# chown -R mogilefs.mogilefs /data/mogilefs/拷贝脚本：[root@master2 mogilefs]# scp /etc/rc.d/init.d/mogstored master3:/etc/rc.d/init.d/root@master3's password: mogstored                                                             100% 1456     1.4KB/s   00:00    [root@master2 mogilefs]# scp /etc/rc.d/init.d/mogstored master4:/etc/rc.d/init.d/root@master4's password: mogstored                                                             100% 1456     1.4KB/s   00:00    [root@master2 mogilefs]# [root@master2 mogilefs]# scp /etc/mogilefs/mogstored.conf master3:/etc/mogilefs/root@master3's password: mogstored.conf                                                        100%   95     0.1KB/s   00:00    [root@master2 mogilefs]# scp /etc/mogilefs/mogstored.conf master4:/etc/mogilefs/root@master4's password: mogstored.conf                                                        100%   95     0.1KB/s   00:00    [root@master2 mogilefs]# 启动服务：[root@master3 ~]# service mogstored startStarting mogstored (via systemctl):                        [  OK  ][root@master4 ~]# service mogstored startStarting mogstored (via systemctl):                        [  OK  ]

5、tracker节点添加storage节点

5.1 检查当前tracker节点信息

[root@master2 ~]# mogadm checkChecking trackers...  127.0.0.1:7001 ... OKChecking hosts...No devices found on tracker(s).[root@master2 ~]# mogadm --trackers=10.201.106.132:7001 checkChecking trackers...  10.201.106.132:7001 ... OKChecking hosts...No devices found on tracker(s).

5.2 添加主机

[root@master2 ~]# mogadm --trackers=10.201.106.132:7001 host add 10.201.106.132 --ip=10.201.106.132 --status=alive[root@master2 ~]# mogadm host list10.201.106.132 [1]: alive  IP:       10.201.106.132:7500添加另外两个节点：[root@master2 ~]# mogadm --trackers=10.201.106.132:7001 host add 10.201.106.133 --ip=10.201.106.133 --status=alive[root@master2 ~]# [root@master2 ~]# mogadm --trackers=10.201.106.132:7001 host add 10.201.106.134 --ip=10.201.106.134 --status=alive[root@master2 ~]# mogadm host list10.201.106.132 [1]: alive  IP:       10.201.106.132:750010.201.106.133 [2]: alive  IP:       10.201.106.133:750010.201.106.134 [3]: alive  IP:       10.201.106.134:7500[root@master2 ~]#

5.3 修改节点状态

[root@master2 ~]# mogadm host mark 10.201.106.133 down[root@master2 ~]# mogadm host list10.201.106.132 [1]: alive  IP:       10.201.106.132:750010.201.106.133 [2]: down  IP:       10.201.106.133:750010.201.106.134 [3]: alive  IP:       10.201.106.134:7500[root@master2 ~]# mogadm host mark 10.201.106.133 alive[root@master2 ~]# mogadm host list10.201.106.132 [1]: alive  IP:       10.201.106.132:750010.201.106.133 [2]: alive  IP:       10.201.106.133:750010.201.106.134 [3]: alive  IP:       10.201.106.134:7500[root@master2 ~]#

6、

6.1 查看所有设备状态

[root@master2 ~]# mogadm device list10.201.106.132 [1]: alive                    used(G)    free(G)   total(G)  weight(%)10.201.106.133 [2]: alive                    used(G)    free(G)   total(G)  weight(%)10.201.106.134 [3]: alive                    used(G)    free(G)   total(G)  weight(%)

6.2 添加设备

[root@master2 ~]# mogadm device add 10.201.106.132 1[root@master2 ~]# mogadm device list10.201.106.132 [1]: alive                    used(G)    free(G)   total(G)  weight(%)   dev1:   alive      0.054     22.864     22.918        10010.201.106.133 [2]: alive                    used(G)    free(G)   total(G)  weight(%)10.201.106.134 [3]: alive                    used(G)    free(G)   total(G)  weight(%)[root@master2 ~]# [root@master2 ~]# mogadm device add 10.201.106.133 2[root@master2 ~]# mogadm device add 10.201.106.134 3[root@master2 ~]# mogadm device list10.201.106.132 [1]: alive                    used(G)    free(G)   total(G)  weight(%)   dev1:   alive      0.055     22.863     22.918        10010.201.106.133 [2]: alive                    used(G)    free(G)   total(G)  weight(%)   dev2:   alive      0.054     22.864     22.918        10010.201.106.134 [3]: alive                    used(G)    free(G)   total(G)  weight(%)   dev3:   alive      0.457     22.461     22.918        100[root@master2 ~]#

6.3 查看现在的状态

[root@master2 ~]# mogadm checkChecking trackers...  127.0.0.1:7001 ... OKChecking hosts...  [ 1] 10.201.106.132 ... OK  [ 2] 10.201.106.133 ... OK  [ 3] 10.201.106.134 ... OKChecking devices...  host device         size(G)    used(G)    free(G)   use%   ob state   I/O%  ---- ------------ ---------- ---------- ---------- ------ ---------- -----  [ 1] dev1            22.918      0.055     22.864   0.24%  writeable   0.1  [ 2] dev2            22.918      0.055     22.864   0.24%  writeable   0.0  [ 3] dev3            22.918      0.458     22.460   2.00%  writeable   0.0  ---- ------------ ---------- ---------- ---------- ------             total:    68.755      0.568     68.188   0.83%[root@master2 ~]#

7、创建管理域

7.1 创建管理域

[root@master2 ~]# mogadm domain add images[root@master2 ~]# mogadm domain add file[root@master2 ~]# mogadm domain list domain               class                mindevcount   replpolicy   hashtype-------------------- -------------------- ------------- ------------ ------- file                 default                   2        MultipleHosts() NONE    images               default                   2        MultipleHosts() NONE   [root@master2 ~]#

8 管理域中的类

8.1 查看class

[root@master2 ~]# mogadm class list domain               class                mindevcount   replpolicy   hashtype-------------------- -------------------- ------------- ------------ ------- file                 default                   2        MultipleHosts() NONE    images               default                   2        MultipleHosts() NONE   [root@master2 ~]#

8.2 在图片域中添加class

[root@master2 ~]# mogadm class add images jpeg[root@master2 ~]# mogadm class add images png[root@master2 ~]# mogadm class add images gif[root@master2 ~]# mogadm class list domain               class                mindevcount   replpolicy   hashtype-------------------- -------------------- ------------- ------------ ------- file                 default                   2        MultipleHosts() NONE    images               default                   2        MultipleHosts() NONE    images               gif                       2        MultipleHosts() NONE    images               jpeg                      2        MultipleHosts() NONE    images               png                       2        MultipleHosts() NONE   [root@master2 ~]#

8.3 class限制，不要副本

[root@master2 ~]# mogadm class add file plaintext --mindevcount=1[root@master2 ~]# mogadm class list domain               class                mindevcount   replpolicy   hashtype-------------------- -------------------- ------------- ------------ ------- file                 default                   2        MultipleHosts() NONE    file                 plaintext                 1        MultipleHosts() NONE

8.4 设置class至少3个保留副本

[root@master2 ~]# mogadm class add file html --replpolicy="MultipleHosts(3)"[root@master2 ~]# mogadm class list domain               class                mindevcount   replpolicy   hashtype-------------------- -------------------- ------------- ------------ ------- file                 default                   2        MultipleHosts() NONE    file                 html                      2        MultipleHosts(3) NONE

9、文件上传

9.1 上传文件

[root@master2 ~]# mogupload --trackers=10.201.106.132:7001 --domain=file --key='/fstab.txt' --file='/etc/fstab' --class=plaintext

9.1 查看文件信息

[root@master2 ~]# mogfileinfo Usage: /usr/bin/mogfileinfo --trackers=host --domain=foo --key='/hello.jpg'[root@master2 ~]# mogfileinfo --trackers=10.201.106.132:7001 --domain=file --key='/fstab.txt'- file: /fstab.txt     class:            plaintext  devcount:                    1    domain:                 file       fid:                    3       key:           /fstab.txt    length:                 1034 - http://10.201.106.132:7500/dev1/0/000/000/0000000003.fid[root@master2 ~]# mogupload --trackers=10.201.106.132:7001 --domain=images --key='/1.jpg' --file='/root/1.jpg' --class=jpeg[root@master2 ~]# mogfileinfo --trackers=10.201.106.132:7001 --domain=images --key='/1.jpg'- file: /1.jpg     class:                 jpeg  devcount:                    1    domain:               images       fid:                    4       key:               /1.jpg    length:                 8264 - http://10.201.106.132:7500/dev1/0/000/000/0000000004.fid

9.2 访问测试

9.3 上传图片

[root@master2 ~]# mogupload --trackers=10.201.106.132:7001 --domain=images --class=jpeg --key='/3.jpg' --file='/root/3.jpg'查看图片信息[root@master2 ~]# mogfileinfo --trackers=10.201.106.132:7001 --domain=images --key='/3.jpg'- file: /3.jpg     class:                 jpeg  devcount:                    1    domain:               images       fid:                    5       key:               /3.jpg    length:                18235 - http://10.201.106.134:7500/dev3/0/000/000/0000000005.fid

10 显示keys

10.1 显示images域下所有keys

[root@master2 ~]# moglistkeys --trackers=10.201.106.132:7001 --domain=images/1.jpg/2.jpg/3.jpg[root@master2 ~]#

10.2 列出ID

[root@master2 ~]# moglistfids --trackers=10.201.106.132:7001 fid 3class plaintextdevcount 1domain filekey /fstab.txtlength 1034fid 4class jpegdevcount 1domain imageskey /1.jpglength 8264

11、查看状态、统计数据

mogstats：mysql客户端，连接mysql[root@master2 ~]# mogstats --db_dsn="DBI:mysql:mogilefs:host=10.201.106.132" --db_user="moguser" --db_pass="mogpass" --verbose --stats="devices,files"Fetching statistics... (devices,files)... files stats...... done... per-device stats...... doneStatistics for devices...  device     host                   files     status  ---------- ---------------- ------------ ----------  dev1       10.201.106.132          2      alive  dev3       10.201.106.134          2      alive  ---------- ---------------- ------------ ----------Statistics for files...  domain               class           files    size (m)  fullsize (m)  -------------------- ----------- ---------- ----------- -------------  file                 plaintext           1           0             0  images               jpeg                3           0             0  -------------------- ----------- ---------- ----------- -------------done

11.1 显示所有状态

[root@master2 ~]# mogstats --db_dsn="DBI:mysql:mogilefs:host=10.201.106.132" --db_user="moguser" --db_pass="mogpass" --verbose --stats="all"

12、之前，不能复制问题

12.1 删除文件

[root@master2 ~]# mogdelete --trackers=10.201.106.132:7001 --domain=images --key='/1.jpg'[root@master2 ~]# mogdelete --trackers=10.201.106.132:7001 --domain=images --key='/2.jpg'[root@master2 ~]# mogdelete --trackers=10.201.106.132:7001 --domain=images --key='/3.jpg'[root@master2 ~]# mogdelete --trackers=10.201.106.132:7001 --domain=file --key='/fstab.txt'

12.2 删除类

[root@master2 ~]# mogadm class delete images jpeg[root@master2 ~]# mogadm class delete images png[root@master2 ~]# mogadm class delete images gif[root@master2 ~]# mogadm class delete file plaintext[root@master2 ~]# mogadm class delete file html[root@master2 ~]# mogadm class list domain               class                mindevcount   replpolicy   hashtype-------------------- -------------------- ------------- ------------ ------- file                 default                   2        MultipleHosts() NONE    images               default                   2        MultipleHosts() NONE   [root@master2 ~]#

12.3 删除域

[root@master2 ~]# mogadm domain delete file[root@master2 ~]# mogadm domain delete images

12.4 重新创建

[root@master2 ~]# mogadm domain add imgs[root@master2 ~]# mogadm domain list domain               class                mindevcount   replpolicy   hashtype-------------------- -------------------- ------------- ------------ ------- imgs                 default                   2        MultipleHosts() NONE   [root@master2 ~]# 上传文件测试：[root@master2 ~]# mogupload --trackers=10.201.106.132:7001 --domain=imgs --key='/1.jpg' --file='/root/1.jpg'还是一个：[root@master2 ~]# mogfileinfo --tracker=10.201.106.132:7001 --domain=imgs --key='/1.jpg'- file: /1.jpg     class:              default  devcount:                    1    domain:                 imgs       fid:                    7       key:               /1.jpg    length:                 8264 - http://10.201.106.133:7500/dev2/0/000/000/0000000007.fid

添加新类：

[root@master2 ~]# mogadm class add imgs jpeg --mindevcount=2 --replpolicy="MultipleHosts(2)"

[root@master2 ~]# mogupload --trackers=10.201.106.132:7001 --domain=imgs --class=jpeg --key='/2.jpg' --file='/root/2.jpg'