Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 0a34c82

Browse files
platinumhamburgplatinumhamburg
committed
[flink][client] refine orphan files cleanup
1 parent 8f86475 commit 0a34c82

23 files changed

Lines changed: 356 additions & 124 deletions

File tree

‎fluss-client/src/main/java/org/apache/fluss/client/admin/Admin.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -782,8 +782,11 @@ CompletableFuture<RegisterResult> registerProducerOffsets(
782782
* @return per-bucket manifest paths and end offsets
783783
*/
784784
@Internal
785-
CompletableFuture<ListRemoteLogManifestsResponse> listRemoteLogManifests(
786-
long tableId, @Nullable Long partitionId);
785+
default CompletableFuture<ListRemoteLogManifestsResponse> listRemoteLogManifests(
786+
long tableId, @Nullable Long partitionId) {
787+
throw new UnsupportedOperationException(
788+
"listRemoteLogManifests is not supported by this Admin implementation");
789+
}
787790

788791
/**
789792
* List per-bucket active KV snapshot ids for a table or partition scope.
@@ -793,6 +796,9 @@ CompletableFuture<ListRemoteLogManifestsResponse> listRemoteLogManifests(
793796
* @return per-bucket active snapshot entries
794797
*/
795798
@Internal
796-
CompletableFuture<ListKvSnapshotsResponse> listKvSnapshots(
797-
long tableId, @Nullable Long partitionId);
799+
default CompletableFuture<ListKvSnapshotsResponse> listKvSnapshots(
800+
long tableId, @Nullable Long partitionId) {
801+
throw new UnsupportedOperationException(
802+
"listKvSnapshots is not supported by this Admin implementation");
803+
}
798804
}

‎fluss-common/src/test/java/org/apache/fluss/fs/FileStatusTest.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.fluss.fs;
19+
20+
import org.junit.jupiter.api.Test;
21+
22+
import static org.assertj.core.api.Assertions.assertThat;
23+
24+
/** Tests for default methods of {@link FileStatus}. */
25+
class FileStatusTest {
26+
27+
/**
28+
* An implementation that does not override {@link FileStatus#getModificationTime()} must
29+
* inherit the fail-safe default of {@link Long#MAX_VALUE}, so time-based filters treat the file
30+
* as "always fresh" and never delete it when modification time is unavailable.
31+
*/
32+
@Test
33+
void defaultModificationTimeIsMaxValueFailSafe() {
34+
FileStatus status =
35+
new FileStatus() {
36+
@Override
37+
public long getLen() {
38+
return 0L;
39+
}
40+
41+
@Override
42+
public boolean isDir() {
43+
return false;
44+
}
45+
46+
@Override
47+
public FsPath getPath() {
48+
return new FsPath("/tmp/x");
49+
}
50+
};
51+
52+
assertThat(status.getModificationTime()).isEqualTo(Long.MAX_VALUE);
53+
}
54+
}

‎fluss-flink/fluss-flink-1.19/src/test/java/org/apache/fluss/flink/action/orphan/Flink19OrphanFilesCleanITCase.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.fluss.flink.action.orphan;
20+
21+
/** The IT case for orphan files cleanup in Flink 1.19. */
22+
class Flink19OrphanFilesCleanITCase extends OrphanFilesCleanITCase {}

‎fluss-flink/fluss-flink-1.20/src/test/java/org/apache/fluss/flink/action/orphan/Flink20OrphanFilesCleanITCase.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.fluss.flink.action.orphan;
20+
21+
/** The IT case for orphan files cleanup in Flink 1.20. */
22+
class Flink20OrphanFilesCleanITCase extends OrphanFilesCleanITCase {}

‎fluss-flink/fluss-flink-2.2/src/main/java/org/apache/fluss/flink/adapter/MultipleParameterToolAdapter.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919

2020
import org.apache.flink.util.MultipleParameterTool;
2121

22+
import javax.annotation.Nullable;
23+
24+
import java.util.Collection;
2225
import java.util.Map;
2326

2427
/**
@@ -43,4 +46,23 @@ public static MultipleParameterToolAdapter fromArgs(String[] args) {
4346
public Map<String, String> toMap() {
4447
return this.multipleParameterTool.toMap();
4548
}
49+
50+
/** Returns whether the given key is present in the parsed arguments. */
51+
public boolean has(String key) {
52+
return this.multipleParameterTool.has(key);
53+
}
54+
55+
/** Returns the value for the given key, or {@code null} if the key is not found. */
56+
@Nullable
57+
public String get(String key) {
58+
return this.multipleParameterTool.get(key);
59+
}
60+
61+
/**
62+
* Returns all values associated with the given key, or {@code null} if the key is not found.
63+
*/
64+
@Nullable
65+
public Collection<String> getMultiParameter(String key) {
66+
return this.multipleParameterTool.getMultiParameter(key);
67+
}
4668
}

‎fluss-flink/fluss-flink-2.2/src/test/java/org/apache/fluss/flink/action/orphan/Flink22OrphanFilesCleanITCase.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.fluss.flink.action.orphan;
20+
21+
/** The IT case for orphan files cleanup in Flink 2.2. */
22+
class Flink22OrphanFilesCleanITCase extends OrphanFilesCleanITCase {}

‎fluss-flink/fluss-flink-action/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
<groupId>org.apache.fluss</groupId>
4242
<artifactId>fluss-flink-common</artifactId>
4343
<version>${project.version}</version>
44+
<scope>provided</scope>
4445
</dependency>
4546

4647
<dependency>

‎fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionFactory.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@
1818
package org.apache.fluss.flink.action;
1919

2020
import org.apache.fluss.annotation.Internal;
21-
22-
import org.apache.flink.api.java.utils.MultipleParameterTool;
21+
import org.apache.fluss.flink.adapter.MultipleParameterToolAdapter;
2322

2423
import java.util.Optional;
2524

@@ -34,7 +33,7 @@ public interface ActionFactory {
3433
String identifier();
3534

3635
/** Construct the action from parsed CLI parameters. Empty when {@code --help} is requested. */
37-
Optional<Action> create(MultipleParameterTool params);
36+
Optional<Action> create(MultipleParameterToolAdapter params);
3837

3938
/** Help text printed when {@code --help} is passed. */
4039
default String help() {

‎fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/ActionLoader.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@
1818
package org.apache.fluss.flink.action;
1919

2020
import org.apache.fluss.annotation.Internal;
21-
22-
import org.apache.flink.api.java.utils.MultipleParameterTool;
21+
import org.apache.fluss.flink.adapter.MultipleParameterToolAdapter;
2322

2423
import java.util.Arrays;
2524
import java.util.Optional;
@@ -56,7 +55,7 @@ public static Optional<Action> createAction(String[] args) {
5655
+ args[0]
5756
+ ". Run with --help for available actions."));
5857
String[] remaining = Arrays.copyOfRange(args, 1, args.length);
59-
MultipleParameterTool params = MultipleParameterTool.fromArgs(remaining);
58+
MultipleParameterToolAdapter params = MultipleParameterToolAdapter.fromArgs(remaining);
6059
if (params.has("help")) {
6160
System.out.println(factory.help());
6261
return Optional.empty();

‎fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/action/orphan/OrphanFilesCleanActionFactory.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,7 @@
2121
import org.apache.fluss.flink.action.Action;
2222
import org.apache.fluss.flink.action.ActionFactory;
2323
import org.apache.fluss.flink.action.orphan.config.OrphanCleanConfig;
24-
25-
import org.apache.flink.api.java.utils.MultipleParameterTool;
24+
import org.apache.fluss.flink.adapter.MultipleParameterToolAdapter;
2625

2726
import java.util.Optional;
2827

@@ -36,7 +35,7 @@ public String identifier() {
3635
}
3736

3837
@Override
39-
public Optional<Action> create(MultipleParameterTool params) {
38+
public Optional<Action> create(MultipleParameterToolAdapter params) {
4039
return Optional.<Action>of(
4140
new OrphanFilesCleanAction(OrphanCleanConfig.fromParams(params)));
4241
}

0 commit comments

Comments
 (0)