A-Zero

# MinIO

官网上说是世界上最快的对象存储。其开源协议是AGPLv3。也就是说，商用要收费，个人若将MinIO用在开源项目上是不收费的。

# MinIO部署

MinIO有多种部署方式，docker部署，简单又方便

# 拉取镜像
docker pull minio/minio

# 创建并启动容器
# 9000是minio服务端口，用于服务的链接和请求；9090是minio客户端端口，用于访问管理界面
docker run -p 9000:9000 -p 9090:9090 \
 --name minio \
 -d --restart=always \
 # 设置minio登录名，不少于3个字符
 -e "MINIO_ACCESS_KEY=admin" \
 # 设置minio登录密码，不少于8个字符
 -e "MINIO_SECRET_KEY=admin123" \
 # minio上传的文件默认存储在容器中的/data目录下
 -v /mydata/minio/data:/data \
 # 启动容器
 minio/minio server \
 # --console-address 指定客户端端口；-address 指定服务端端口
 /data --console-address ":9090" -address ":9000"

访问http://ip:9090，出现登录页面则部署成功

# SpringBoot整合MinIO

  • 创建一个Maven项目，添加坐标
<!--web-->
<dependency>
	<groupId>org.springframework.boot</groupId>
	<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!--minio-->
<dependency>
	<groupId>io.minio</groupId>
	<artifactId>minio</artifactId>
	<version>8.4.3</version>
</dependency>
  • SpringBoot启动类搞起来
package com.nanfeng.minio;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
public class App {
    public static void main(String[] args) {
        SpringApplication.run(App.class);
    }
}
  • 创建application.properties
## minio 的UTL
minio.endpoint=http://localhost:9000
## minio 的账号密码
minio.accessKey=admin
minio.secretKey=admin123
## minio 的自己建的桶
minio.bucket.test=test
  • 创建MinIO的配置类 并放在config包下面
@Configuration
public class MinioConfig {

    //读取参数
    @Value("${minio.endpoint}")
    private String endpoint;
    @Value("${minio.accessKey}")
    private String accessKey;
    @Value("${minio.secretKey}")
    private String secretKey;

    @Bean
    public MinioClient minioClient() {

        MinioClient minioClient =
                MinioClient.builder()
                        .endpoint(endpoint)
                        .credentials(accessKey, secretKey)
                        .build();
        return minioClient;
    }
}

实际上minioClient自带的API很丰富，到这一步就ok了。下面是锦上添花的功能

  • 弄一个操作类，相当于一个工具包
package com.nanfeng.minio.service;

import io.minio.*;
import io.minio.errors.*;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.io.IOException;
import java.io.InputStream;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;

/**
 * minio 操作类
 */
@Service
public class MinIOService {

    // 我最喜欢用@Resource 强烈建议不要用@autowire
    @Resource
    private MinioClient minioClient;

    @Value("${minio.bucket.test}")
    private String bucket;

    @Value("${minio.endpoint}")
    private String endpoint;

    /**
     * 本地文件上传
     * @param localPath 本地路径
     * @param remotePath 远程路径
     * @return 可访问地址
     */
    public String upload(String localPath,String remotePath) throws IOException, ServerException, InsufficientDataException, ErrorResponseException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException {
            UploadObjectArgs uploadObjectArgs = UploadObjectArgs.builder()
                    .bucket(bucket)
                    .object(localPath)
                    .filename(remotePath)
                    .build();
            minioClient.uploadObject(uploadObjectArgs);
            return endpoint + "/" + bucket + "/" + remotePath;
    }

    /**
     * 用流上传
     * @param is 文件流
     * @param remotePath 远程路径
     * @return 可访问地址
     */
    public String upload(InputStream is, String remotePath) throws ServerException, InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException {
        PutObjectArgs args = PutObjectArgs.builder()
                .bucket(bucket)
                .object(remotePath)
                .stream(is, -1, 10485760)
                .build();
        minioClient.putObject(args);
        return endpoint + "/" + bucket + "/" + remotePath;
    }

    /**
     * 删除文件
     * @param remotePath 远程路径
     * @return
     */
    public void delete(String remotePath) throws ServerException, InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException {
        RemoveObjectArgs args = RemoveObjectArgs.builder()
                .bucket(bucket)
                .object(remotePath)
                .build();
        minioClient.removeObject(args);
    }

    /**
     * 获取流
     * @param remotePath 远程路径
     */
    public InputStream getInputStream(String remotePath) throws ServerException, InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException {
        GetObjectArgs args = GetObjectArgs.builder()
                .bucket(bucket)
                .object(remotePath)
                .build();
        return minioClient.getObject(args);
    }
}
  • 写一个controller用来测试一下
package com.nanfeng.minio.controller;


import com.nanfeng.minio.service.MinIOService;
import io.minio.errors.*;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;

import javax.annotation.Resource;
import java.io.IOException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;

@RestController
public class MinIOController {

    @Resource
    private MinIOService minIOService;

    @PostMapping("/upload")
    public HashMap<String, String> upload(@RequestParam(name = "file", required = false) MultipartFile file) throws IOException, ServerException, InsufficientDataException, ErrorResponseException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException {
        String originalFilename = file.getOriginalFilename();
        assert originalFilename != null;
        String fileName = System.currentTimeMillis() + originalFilename.substring(originalFilename.lastIndexOf("."));
        String url = minIOService.upload(file.getInputStream(), fileName);
        HashMap<String, String> map = new HashMap<String, String>();
        map.put("url",url);
        map.put("fileName",fileName);
        return map;
    }

    @GetMapping("/remove")
    public String remove(String fileName) throws ServerException, InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException {
        minIOService.delete(fileName);
        return "success";
    }
}

# FastDFS

FastDFS是一个开源的轻量级分布式文件系统，它对文件进行管理，功能包括：文件存储、文件同步、文件访问（文件上传、文件下载）等，解决了大容量存储和负载均衡的问题。

FastDFS中的文件标识分为两个部分：卷名和文件名，二者缺一不可。

# FastDFS主要内容

  • tracker：追踪者服务器，主要用于协调调度，可以起到负载均衡的作用，记录storage的相关状态信息
  • storage：存储服务器，用于保存文件以及文件的元数据信息
  • group：组，同组节点提供冗余备份，不同组用于扩容
  • mata data：文件的元数据信息，比如长宽信息，图片后缀，视频的帧数等

# FastDFS安装

  • 拉取镜像
docker pull delron/fastdfs
  • 构建tracker容器，端口为22122
docker run -d --name tracker -v /mydata/fastdfs/tracker:/var/fdfs -p 22122:22122  delron/fastdfs tracker
  • 构建storage容器，端口为23000，TRACKER_SERVER地址不要使用127.0.0.1
docker run -d --name storage -e TRACKER_SERVER=192.168.0.172:22122 -v /mydata/fastdfs/storage:/var/fdfs -p 23000:23000  delron/fastdfs storage
  • 配置http访问的端口，默认情况下在Storage服务中是帮我们安装了Nginx服务的，端口为：8888
# 进入storage容器，docker exec -it storage bash，配置文件在/etc/fdfs目录下的storage.conf中

# 将配置文件拷贝出来
docker cp storage:/fdfs_conf/storage.conf  ./

# 配置文件修改后再拷贝到容器内
docker cp ./storage.conf storage:/fdfs_conf/

# 需要修改的地方，将nginx端口改为正在使用的
http.server_port=8888

# nginx配置文件nginx.conf在/usr/local/nginx/conf目录下

注意

如果宿主机也安装了nginx，就让宿主机端口和storage中nginx端口不一样，这样就相当于装了两个nginx，如果端口有限，可以利用宿主机nginx配置反向代理proxy_pass代理到storage中nginx，配置proxy_pass时不要使用127.0.0.1，会无法代理成功，使用本地ip即可

# 测试上传文件

  • 使用web模块进行文件的上传，将文件上传至FastDFS文件系统
  • 将一张照片（test.png）放置在/mydata/fastdfs/storage目录下，进入storage容器，进入/var/fdfs目录，运行下面命令：
/usr/bin/fdfs_upload_file /etc/fdfs/client.conf test.jpg
  • 此时将该图片已上传至文件系统，并在执行该语句后返回图片存储的uri：
通过url访问http://ip:8888/group1/M00/00/00/CgACD1z7SEuAXrIqAA1eBLGVLow043.png，即可查看到图片

# SpringBoot集成FastDFS

  • 引入依赖
<dependency>
    <groupId>net.oschina.zcx7878</groupId>
    <artifactId>fastdfs-client-java</artifactId>
    <version>1.27.0.0</version>
</dependency>
  • 配置tracker.conf文件
connect_timeout = 60 # 连接超时时间，单位为秒
network_timeout = 60 # 通信超时时间，单位为秒。
charset = UTF-8 # 字符集
http.tracker_http_port = 8080 # tracker的http端口
tracker_server = ip:22122 # tracker服务器IP和端口设置
  • 文件信息封装
@Data
public class FastDFSFile {
    //文件名称
    private String name;
    //文件内容
    private byte[] content;
    //文件扩展名
    private String ext;
    //文件MD5摘要值
    private String md5;
    //文件创建作者
    private String author;

    public FastDFSFile(String name, byte[] content, String ext, String height, String width, String author) {
        super();
        this.name = name;
        this.content = content;
        this.ext = ext;
        this.author = author;
    }

    public FastDFSFile(String name, byte[] content, String ext) {
        super();
        this.name = name;
        this.content = content;
        this.ext = ext;
    }

    public FastDFSFile(String name, byte[] content, String ext, String md5, String author) {
        this.name = name;
        this.content = content;
        this.ext = ext;
        this.md5 = md5;
        this.author = author;
    }
}
  • 文件操作工具类
@Slf4j
public class FastDFSClient {

    /***
     * 初始化加载FastDFS的TrackerServer配置
     */
    static {
        try {
            String filePath = new ClassPathResource("fdfs_client.conf").getFile().getAbsolutePath();
            ClientGlobal.init(filePath);
        } catch (Exception e) {
            log.error("FastDFS Client Init Fail!", e);
        }
    }

    /***
     * 文件上传
     * @param file
     * @return 1.文件的组名  2.文件的路径信息
     */
    public static String[] upload(FastDFSFile file) {
        //获取文件的作者
        NameValuePair[] meta_list = new NameValuePair[1];
        meta_list[0] = new NameValuePair("author", file.getAuthor());

        //接收返回数据
        String[] uploadResults = null;
        StorageClient storageClient = null;
        try {
            //创建StorageClient客户端对象
            storageClient = getTrackerClient();

            /***
             * 文件上传
             * 1)文件字节数组
             * 2)文件扩展名
             * 3)文件作者
             */
            uploadResults = storageClient.upload_file(file.getContent(), file.getExt(), meta_list);
        } catch (Exception e) {
            log.error("Exception when uploadind the file:" + file.getName(), e);
        }

        if (uploadResults == null && storageClient != null) {
            log.error("upload file fail, error code:" + storageClient.getErrorCode());
        }
        //获取组名
        String groupName = uploadResults[0];
        //获取文件存储路径
        String remoteFileName = uploadResults[1];
        return uploadResults;
    }

    /***
     * 获取文件信息
     * @param groupName:组名
     * @param remoteFileName：文件存储完整名
     * @return
     */
    public static FileInfo getFile(String groupName, String remoteFileName) {
        try {
            StorageClient storageClient = getTrackerClient();
            return storageClient.get_file_info(groupName, remoteFileName);
        } catch (Exception e) {
            log.error("Exception: Get File from Fast DFS failed", e);
        }
        return null;
    }

    /***
     * 文件下载
     * @param groupName
     * @param remoteFileName
     * @return
     */
    public static InputStream downFile(String groupName, String remoteFileName) {
        try {
            //创建StorageClient
            StorageClient storageClient = getTrackerClient();

            //下载文件
            byte[] fileByte = storageClient.download_file(groupName, remoteFileName);
            InputStream ins = new ByteArrayInputStream(fileByte);
            return ins;
        } catch (Exception e) {
            log.error("Exception: Get File from Fast DFS failed", e);
        }
        return null;
    }

    /***
     * 文件删除
     * @param groupName
     * @param remoteFileName
     * @throws Exception
     */
    public static void deleteFile(String groupName, String remoteFileName)
            throws Exception {
        //创建StorageClient
        StorageClient storageClient = getTrackerClient();

        //删除文件
        int i = storageClient.delete_file(groupName, remoteFileName);
    }

    /***
     * 获取Storage组
     * @param groupName
     * @return
     * @throws IOException
     */
    public static StorageServer[] getStoreStorages(String groupName)
            throws IOException {
        //创建TrackerClient
        TrackerClient trackerClient = new TrackerClient();
        //获取TrackerServer
        TrackerServer trackerServer = trackerClient.getConnection();
        //获取Storage组
        return trackerClient.getStoreStorages(trackerServer, groupName);
    }

    /***
     * 获取Storage信息,IP和端口
     * @param groupName
     * @param remoteFileName
     * @return
     * @throws IOException
     */
    public static ServerInfo[] getFetchStorages(String groupName,
                                                String remoteFileName) throws IOException {
        TrackerClient trackerClient = new TrackerClient();
        TrackerServer trackerServer = trackerClient.getConnection();
        return trackerClient.getFetchStorages(trackerServer, groupName, remoteFileName);
    }

    /***
     * 获取Tracker服务地址
     * @return
     * @throws IOException
     */
    public static String getTrackerUrl() throws IOException {
        return "http://" + getTrackerServer().getInetSocketAddress().getHostString() + ":" + ClientGlobal.getG_tracker_http_port() + "/";
    }

    /***
     * 获取Storage客户端
     * @return
     * @throws IOException
     */
    private static StorageClient getTrackerClient() throws IOException {
        TrackerServer trackerServer = getTrackerServer();
        StorageClient storageClient = new StorageClient(trackerServer, null);
        return storageClient;
    }

    /***
     * 获取Tracker
     * @return
     * @throws IOException
     */
    private static TrackerServer getTrackerServer() throws IOException {
        TrackerClient trackerClient = new TrackerClient();
        TrackerServer trackerServer = trackerClient.getConnection();
        return trackerServer;
    }
}
  • 文件上传
@RestController
@RequestMapping("/file")
public class FileController {

    @PostMapping("/upload")
    public Result uploadFile(MultipartFile file) {
        try {
            //判断文件是否存在
            if (file == null) {
                throw new BusinessException("文件不存在");
            }
            //获取文件的完整名称
            String originalFilename = file.getOriginalFilename();
            if (StringUtils.isEmpty(originalFilename)) {
                throw new BusinessException("文件不存在");
            }

            //获取文件的扩展名称  abc.jpg   jpg
            String extName = originalFilename.substring(originalFilename.lastIndexOf(".") + 1);

            //获取文件内容
            byte[] content = file.getBytes();

            //创建文件上传的封装实体类
            FastDFSFile fastDFSFile = new FastDFSFile(originalFilename, content, extName);

            //基于工具类进行文件上传,并接受返回参数  String[]
            String[] uploadResult = FastDFSClient.upload(fastDFSFile);

            //封装返回结果
            String url = FastDFSClient.getTrackerUrl() + uploadResult[0] + "/" + uploadResult[1];
            return new Result(true, StatusCode.OK, "文件上传成功", url);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return new Result(false, StatusCode.ERROR, "文件上传失败");
    }
}

← 服务监控 →