Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 1545733

Browse files
KFCFansKFCFans
committed
Merge branch 'pr-Minio' into 4.3.6
2 parents 882392a + fa8b4f2 commit 1545733

File tree

5 files changed

+255
-6
lines changed

5 files changed

+255
-6
lines changed

‎README_zhCN.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,4 +69,6 @@ PowerJob 的设计目标为企业级的分布式任务调度平台，即成为
6969
* 欢迎共同参与本项目的贡献，PR和Issue都大大滴欢迎（求求了）～
7070
* 觉得还不错的话，可以点个Star支持一下哦～ =￣ω￣=
7171
* 联系方式@KFCFans -> `[email protected]`
72-
* 用户交流QQ群（因广告信息泛滥，加群需要验证，请认真填写申请原因）：487453839
72+
* 用户交流QQ群（因广告信息泛滥，加群需要验证，请认真填写申请原因）：
73+
* 一群（已满）：487453839
74+
* 二群：834937813

‎powerjob-server/pom.xml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
<powerjob-remote-impl-akka.version>4.3.5</powerjob-remote-impl-akka.version>
5656
<springdoc-openapi-ui.version>1.6.14</springdoc-openapi-ui.version>
5757
<aliyun-sdk-oss.version>3.17.1</aliyun-sdk-oss.version>
58+
<minio.version>8.5.2</minio.version>
5859
<commons-collections4.version>4.4</commons-collections4.version>
5960
</properties>
6061

@@ -113,7 +114,12 @@
113114
<artifactId>aliyun-sdk-oss</artifactId>
114115
<version>${aliyun-sdk-oss.version}</version>
115116
</dependency>
116-
117+
<!-- 存储扩展-Minio，未使用可移除 -->
118+
<dependency>
119+
<groupId>io.minio</groupId>
120+
<artifactId>minio</artifactId>
121+
<version>${minio.version}</version>
122+
</dependency>
117123
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-collections4 -->
118124
<dependency>
119125
<groupId>org.apache.commons</groupId>

‎powerjob-server/powerjob-server-persistence/pom.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@
4040
<groupId>com.aliyun.oss</groupId>
4141
<artifactId>aliyun-sdk-oss</artifactId>
4242
</dependency>
43+
<dependency>
44+
<groupId>io.minio</groupId>
45+
<artifactId>minio</artifactId>
46+
</dependency>
4347
</dependencies>
4448

4549
</project>

‎powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,7 @@
44
import org.springframework.context.annotation.Conditional;
55
import org.springframework.context.annotation.Configuration;
66
import tech.powerjob.server.extension.dfs.DFsService;
7-
import tech.powerjob.server.persistence.storage.impl.AliOssService;
8-
import tech.powerjob.server.persistence.storage.impl.EmptyDFsService;
9-
import tech.powerjob.server.persistence.storage.impl.GridFsService;
10-
import tech.powerjob.server.persistence.storage.impl.MySqlSeriesDfsService;
7+
import tech.powerjob.server.persistence.storage.impl.*;
118

129
/**
1310
* 初始化内置的存储服务
@@ -36,6 +33,12 @@ public DFsService initAliOssFs() {
3633
return new AliOssService();
3734
}
3835

36+
@Bean
37+
@Conditional(MinioOssService.MinioOssCondition.class)
38+
public DFsService initMinioOssFs() {
39+
return new MinioOssService();
40+
}
41+
3942
@Bean
4043
@Conditional(EmptyDFsService.EmptyCondition.class)
4144
public DFsService initEmptyDfs() {

‎powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MinioOssService.java‎

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
package tech.powerjob.server.persistence.storage.impl;
2+
3+
import com.google.common.collect.Lists;
4+
import com.google.common.collect.Maps;
5+
import io.minio.*;
6+
import lombok.SneakyThrows;
7+
import lombok.extern.slf4j.Slf4j;
8+
import org.apache.commons.io.FileUtils;
9+
import org.apache.commons.lang3.StringUtils;
10+
import org.apache.commons.lang3.exception.ExceptionUtils;
11+
import org.springframework.context.ApplicationContext;
12+
import org.springframework.context.annotation.Conditional;
13+
import org.springframework.core.env.Environment;
14+
import tech.powerjob.server.common.spring.condition.PropertyAndOneBeanCondition;
15+
import tech.powerjob.server.extension.dfs.*;
16+
import tech.powerjob.server.persistence.storage.AbstractDFsService;
17+
18+
import javax.annotation.Priority;
19+
import java.nio.file.Files;
20+
import java.util.Date;
21+
import java.util.List;
22+
import java.util.Map;
23+
import java.util.Optional;
24+
25+
/**
26+
* MINIO support
27+
* 配置项：
28+
* oms.storage.dfs.minio.endpoint
29+
* oms.storage.dfs.minio.bucketName
30+
* oms.storage.dfs.minio.accessKey
31+
* oms.storage.dfs.minio.secretKey
32+
*
33+
* @author xinyi
34+
* @since 2023/8/21
35+
*/
36+
@Slf4j
37+
@Priority(value = Integer.MAX_VALUE - 1)
38+
@Conditional(MinioOssService.MinioOssCondition.class)
39+
public class MinioOssService extends AbstractDFsService {
40+
41+
private static final String TYPE_MINIO = "minio";
42+
private static final String KEY_ENDPOINT = "endpoint";
43+
private static final String KEY_BUCKET_NAME = "bucketName";
44+
private static final String ACCESS_KEY = "accessKey";
45+
private static final String SECRET_KEY = "secretKey";
46+
private MinioClient minioClient;
47+
private String bucket;
48+
private static final String NO_SUCH_KEY = "NoSuchKey";
49+
50+
@Override
51+
public void store(StoreRequest storeRequest) {
52+
try {
53+
minioClient.uploadObject(UploadObjectArgs.builder()
54+
.bucket(this.bucket)
55+
.object(parseFileName(storeRequest.getFileLocation()))
56+
.filename(storeRequest.getLocalFile().getPath())
57+
.contentType(Files.probeContentType(storeRequest.getLocalFile().toPath()))
58+
.build());
59+
} catch (Throwable t) {
60+
ExceptionUtils.rethrow(t);
61+
}
62+
}
63+
64+
@Override
65+
public void download(DownloadRequest downloadRequest) {
66+
try {
67+
FileUtils.forceMkdirParent(downloadRequest.getTarget());
68+
// 下载文件
69+
minioClient.downloadObject(
70+
DownloadObjectArgs.builder()
71+
.bucket(this.bucket)
72+
.object(parseFileName(downloadRequest.getFileLocation()))
73+
.filename(downloadRequest.getTarget().getAbsolutePath())
74+
.build());
75+
} catch (Throwable t) {
76+
ExceptionUtils.rethrow(t);
77+
}
78+
}
79+
80+
/**
81+
* 获取文件元
82+
*
83+
* @param fileLocation 文件位置
84+
*/
85+
@Override
86+
public Optional<FileMeta> fetchFileMeta(FileLocation fileLocation) {
87+
try {
88+
StatObjectResponse stat = minioClient.statObject(StatObjectArgs.builder()
89+
.bucket(this.bucket)
90+
.object(parseFileName(fileLocation))
91+
.build());
92+
return Optional.ofNullable(stat).map(minioStat -> {
93+
94+
Map<String, Object> metaInfo = Maps.newHashMap();
95+
if (stat.userMetadata() != null) {
96+
metaInfo.putAll(stat.userMetadata());
97+
}
98+
return new FileMeta()
99+
.setLastModifiedTime(Date.from(stat.lastModified().toInstant()))
100+
.setLength(stat.size())
101+
.setMetaInfo(metaInfo);
102+
});
103+
} catch (Exception oe) {
104+
String errorCode = oe.getMessage();
105+
if (NO_SUCH_KEY.equalsIgnoreCase(errorCode)) {
106+
return Optional.empty();
107+
}
108+
ExceptionUtils.rethrow(oe);
109+
}
110+
return Optional.empty();
111+
}
112+
113+
private static String parseFileName(FileLocation fileLocation) {
114+
return String.format("%s/%s", fileLocation.getBucket(), fileLocation.getName());
115+
}
116+
117+
/**
118+
* 清理过期文件
119+
*
120+
* @param bucket 桶名
121+
* @param days 日期
122+
*/
123+
@Override
124+
public void cleanExpiredFiles(String bucket, int days) {
125+
/*
126+
使用Minio的管理界面或Minio客户端命令行工具设置对象的生命周期规则。在生命周期规则中定义文件的过期时间。Minio将自动根据设置的规则删除过期的文件。
127+
*/
128+
}
129+
130+
/**
131+
* 释放连接
132+
*/
133+
@Override
134+
public void destroy() {
135+
//minioClient.close();
136+
}
137+
138+
/**
139+
* 初始化minio
140+
*
141+
* @param applicationContext /
142+
*/
143+
@Override
144+
protected void init(ApplicationContext applicationContext) {
145+
Environment environment = applicationContext.getEnvironment();
146+
147+
String endpoint = fetchProperty(environment, TYPE_MINIO, KEY_ENDPOINT);
148+
String bucketName = fetchProperty(environment, TYPE_MINIO, KEY_BUCKET_NAME);
149+
String accessKey = fetchProperty(environment, TYPE_MINIO, ACCESS_KEY);
150+
String secretKey = fetchProperty(environment, TYPE_MINIO, SECRET_KEY);
151+
152+
try {
153+
initOssClient(endpoint, bucketName, accessKey, secretKey);
154+
} catch (Exception e) {
155+
ExceptionUtils.rethrow(e);
156+
}
157+
}
158+
159+
/**
160+
* 创建minio连接并且创建桶
161+
*
162+
* @param endpoint 端口
163+
* @param bucketName 桶名
164+
* @param accessKey 访问密钥
165+
* @param secretKey 秘密密钥
166+
*/
167+
public void initOssClient(String endpoint, String bucketName, String accessKey, String secretKey) {
168+
log.info("[Minio] init OSS by config: endpoint={}, bucketName={}, accessKey={}, secretKey={}", endpoint, bucketName, accessKey, secretKey);
169+
if (StringUtils.isEmpty(bucketName)) {
170+
throw new IllegalArgumentException("'oms.storage.dfs.minio.bucketName' can't be empty, please creat a bucket in minio oss console then config it to powerjob");
171+
}
172+
this.bucket = bucketName;
173+
minioClient = MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).build();
174+
createBucket(bucketName);
175+
log.info("[Minio] initialize OSS successfully!");
176+
}
177+
178+
/**
179+
* 创建 bucket
180+
*
181+
* @param bucketName 桶名
182+
*/
183+
@SneakyThrows(Exception.class)
184+
public void createBucket(String bucketName) {
185+
if (!bucketExists(bucketName)) {
186+
minioClient.makeBucket(MakeBucketArgs.builder()
187+
.bucket(bucketName).build());
188+
}
189+
String policy = "{\n" +
190+
" \"Version\": \"2012-10-17\",\n" +
191+
" \"Statement\": [\n" +
192+
" {\n" +
193+
" \"Action\": [\n" +
194+
" \"s3:GetObject\"\n" +
195+
" ],\n" +
196+
" \"Effect\": \"Allow\",\n" +
197+
" \"Principal\": {\n" +
198+
" \"AWS\": [\n" +
199+
" \"*\"\n" +
200+
" ]\n" +
201+
" },\n" +
202+
" \"Resource\": [\n" +
203+
" \"arn:aws:s3:::" + bucketName + "/*\"\n" +
204+
" ]\n" +
205+
" }\n" +
206+
" ]\n" +
207+
"}";
208+
minioClient.setBucketPolicy(SetBucketPolicyArgs.builder().bucket(bucketName).config(policy).build());
209+
}
210+
211+
/**
212+
* 判断 bucket是否存在
213+
*
214+
* @param bucketName: 桶名
215+
* @return boolean
216+
*/
217+
@SneakyThrows(Exception.class)
218+
public boolean bucketExists(String bucketName) {
219+
return minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
220+
}
221+
222+
public static class MinioOssCondition extends PropertyAndOneBeanCondition {
223+
224+
@Override
225+
protected List<String> anyConfigKey() {
226+
return Lists.newArrayList("oms.storage.dfs.minio.endpoint");
227+
}
228+
229+
@Override
230+
protected Class<?> beanType() {
231+
return DFsService.class;
232+
}
233+
}
234+
}

0 commit comments

Comments
 (0)