Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 1e8b598

Browse files
omalleyomalley
committed
ORC-128. Add getStatistics to Writer API to allow user to get statistics as the
file is written. Fixes apache#78 Signed-off-by: Owen O'Malley <[email protected]>
1 parent 0e92c5c commit 1e8b598

File tree

6 files changed

+379
-7
lines changed

6 files changed

+379
-7
lines changed

‎java/core/src/java/org/apache/orc/Writer.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,4 +111,16 @@ public void appendStripe(byte[] stripe, int offset, int length,
111111
* @param userMetadata - user metadata
112112
*/
113113
public void appendUserMetadata(List<OrcProto.UserMetadataItem> userMetadata);
114+
115+
/**
116+
* Get the statistics about the columns in the file. The output of this is
117+
* based on the time at which it is called. It shall use all of the currently
118+
* written data to provide the statistics.
119+
*
120+
* Please note there are costs involved with invoking this method and should
121+
* be used judiciously.
122+
*
123+
* @return the information about the column
124+
*/
125+
ColumnStatistics[] getStatistics() throws IOException;
114126
}

‎java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java‎

Lines changed: 316 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,34 @@
3939

4040
public class ColumnStatisticsImpl implements ColumnStatistics {
4141

42+
@Override
43+
public boolean equals(Object o) {
44+
if (this == o) {
45+
return true;
46+
}
47+
if (!(o instanceof ColumnStatisticsImpl)) {
48+
return false;
49+
}
50+
51+
ColumnStatisticsImpl that = (ColumnStatisticsImpl) o;
52+
53+
if (count != that.count) {
54+
return false;
55+
}
56+
if (hasNull != that.hasNull) {
57+
return false;
58+
}
59+
60+
return true;
61+
}
62+
63+
@Override
64+
public int hashCode() {
65+
int result = (int) (count ^ (count >>> 32));
66+
result = 31 * result + (hasNull ? 1 : 0);
67+
return result;
68+
}
69+
4270
private static final class BooleanStatisticsImpl extends ColumnStatisticsImpl
4371
implements BooleanColumnStatistics {
4472
private long trueCount = 0;
@@ -102,6 +130,34 @@ public long getTrueCount() {
102130
public String toString() {
103131
return super.toString() + " true: " + trueCount;
104132
}
133+
134+
@Override
135+
public boolean equals(Object o) {
136+
if (this == o) {
137+
return true;
138+
}
139+
if (!(o instanceof BooleanStatisticsImpl)) {
140+
return false;
141+
}
142+
if (!super.equals(o)) {
143+
return false;
144+
}
145+
146+
BooleanStatisticsImpl that = (BooleanStatisticsImpl) o;
147+
148+
if (trueCount != that.trueCount) {
149+
return false;
150+
}
151+
152+
return true;
153+
}
154+
155+
@Override
156+
public int hashCode() {
157+
int result = super.hashCode();
158+
result = 31 * result + (int) (trueCount ^ (trueCount >>> 32));
159+
return result;
160+
}
105161
}
106162

107163
private static final class IntegerStatisticsImpl extends ColumnStatisticsImpl
@@ -247,6 +303,50 @@ public String toString() {
247303
}
248304
return buf.toString();
249305
}
306+
307+
@Override
308+
public boolean equals(Object o) {
309+
if (this == o) {
310+
return true;
311+
}
312+
if (!(o instanceof IntegerStatisticsImpl)) {
313+
return false;
314+
}
315+
if (!super.equals(o)) {
316+
return false;
317+
}
318+
319+
IntegerStatisticsImpl that = (IntegerStatisticsImpl) o;
320+
321+
if (minimum != that.minimum) {
322+
return false;
323+
}
324+
if (maximum != that.maximum) {
325+
return false;
326+
}
327+
if (sum != that.sum) {
328+
return false;
329+
}
330+
if (hasMinimum != that.hasMinimum) {
331+
return false;
332+
}
333+
if (overflow != that.overflow) {
334+
return false;
335+
}
336+
337+
return true;
338+
}
339+
340+
@Override
341+
public int hashCode() {
342+
int result = super.hashCode();
343+
result = 31 * result + (int) (minimum ^ (minimum >>> 32));
344+
result = 31 * result + (int) (maximum ^ (maximum >>> 32));
345+
result = 31 * result + (int) (sum ^ (sum >>> 32));
346+
result = 31 * result + (hasMinimum ? 1 : 0);
347+
result = 31 * result + (overflow ? 1 : 0);
348+
return result;
349+
}
250350
}
251351

252352
private static final class DoubleStatisticsImpl extends ColumnStatisticsImpl
@@ -364,6 +464,50 @@ public String toString() {
364464
buf.append(sum);
365465
return buf.toString();
366466
}
467+
468+
@Override
469+
public boolean equals(Object o) {
470+
if (this == o) {
471+
return true;
472+
}
473+
if (!(o instanceof DoubleStatisticsImpl)) {
474+
return false;
475+
}
476+
if (!super.equals(o)) {
477+
return false;
478+
}
479+
480+
DoubleStatisticsImpl that = (DoubleStatisticsImpl) o;
481+
482+
if (hasMinimum != that.hasMinimum) {
483+
return false;
484+
}
485+
if (Double.compare(that.minimum, minimum) != 0) {
486+
return false;
487+
}
488+
if (Double.compare(that.maximum, maximum) != 0) {
489+
return false;
490+
}
491+
if (Double.compare(that.sum, sum) != 0) {
492+
return false;
493+
}
494+
495+
return true;
496+
}
497+
498+
@Override
499+
public int hashCode() {
500+
int result = super.hashCode();
501+
long temp;
502+
result = 31 * result + (hasMinimum ? 1 : 0);
503+
temp = Double.doubleToLongBits(minimum);
504+
result = 31 * result + (int) (temp ^ (temp >>> 32));
505+
temp = Double.doubleToLongBits(maximum);
506+
result = 31 * result + (int) (temp ^ (temp >>> 32));
507+
temp = Double.doubleToLongBits(sum);
508+
result = 31 * result + (int) (temp ^ (temp >>> 32));
509+
return result;
510+
}
367511
}
368512

369513
protected static final class StringStatisticsImpl extends ColumnStatisticsImpl
@@ -498,6 +642,42 @@ public String toString() {
498642
}
499643
return buf.toString();
500644
}
645+
646+
@Override
647+
public boolean equals(Object o) {
648+
if (this == o) {
649+
return true;
650+
}
651+
if (!(o instanceof StringStatisticsImpl)) {
652+
return false;
653+
}
654+
if (!super.equals(o)) {
655+
return false;
656+
}
657+
658+
StringStatisticsImpl that = (StringStatisticsImpl) o;
659+
660+
if (sum != that.sum) {
661+
return false;
662+
}
663+
if (minimum != null ? !minimum.equals(that.minimum) : that.minimum != null) {
664+
return false;
665+
}
666+
if (maximum != null ? !maximum.equals(that.maximum) : that.maximum != null) {
667+
return false;
668+
}
669+
670+
return true;
671+
}
672+
673+
@Override
674+
public int hashCode() {
675+
int result = super.hashCode();
676+
result = 31 * result + (minimum != null ? minimum.hashCode() : 0);
677+
result = 31 * result + (maximum != null ? maximum.hashCode() : 0);
678+
result = 31 * result + (int) (sum ^ (sum >>> 32));
679+
return result;
680+
}
501681
}
502682

503683
protected static final class BinaryStatisticsImpl extends ColumnStatisticsImpl implements
@@ -569,6 +749,34 @@ public String toString() {
569749
}
570750
return buf.toString();
571751
}
752+
753+
@Override
754+
public boolean equals(Object o) {
755+
if (this == o) {
756+
return true;
757+
}
758+
if (!(o instanceof BinaryStatisticsImpl)) {
759+
return false;
760+
}
761+
if (!super.equals(o)) {
762+
return false;
763+
}
764+
765+
BinaryStatisticsImpl that = (BinaryStatisticsImpl) o;
766+
767+
if (sum != that.sum) {
768+
return false;
769+
}
770+
771+
return true;
772+
}
773+
774+
@Override
775+
public int hashCode() {
776+
int result = super.hashCode();
777+
result = 31 * result + (int) (sum ^ (sum >>> 32));
778+
return result;
779+
}
572780
}
573781

574782
private static final class DecimalStatisticsImpl extends ColumnStatisticsImpl
@@ -694,6 +902,42 @@ public String toString() {
694902
}
695903
return buf.toString();
696904
}
905+
906+
@Override
907+
public boolean equals(Object o) {
908+
if (this == o) {
909+
return true;
910+
}
911+
if (!(o instanceof DecimalStatisticsImpl)) {
912+
return false;
913+
}
914+
if (!super.equals(o)) {
915+
return false;
916+
}
917+
918+
DecimalStatisticsImpl that = (DecimalStatisticsImpl) o;
919+
920+
if (minimum != null ? !minimum.equals(that.minimum) : that.minimum != null) {
921+
return false;
922+
}
923+
if (maximum != null ? !maximum.equals(that.maximum) : that.maximum != null) {
924+
return false;
925+
}
926+
if (sum != null ? !sum.equals(that.sum) : that.sum != null) {
927+
return false;
928+
}
929+
930+
return true;
931+
}
932+
933+
@Override
934+
public int hashCode() {
935+
int result = super.hashCode();
936+
result = 31 * result + (minimum != null ? minimum.hashCode() : 0);
937+
result = 31 * result + (maximum != null ? maximum.hashCode() : 0);
938+
result = 31 * result + (sum != null ? sum.hashCode() : 0);
939+
return result;
940+
}
697941
}
698942

699943
private static final class DateStatisticsImpl extends ColumnStatisticsImpl
@@ -815,6 +1059,46 @@ public String toString() {
8151059
}
8161060
return buf.toString();
8171061
}
1062+
1063+
@Override
1064+
public boolean equals(Object o) {
1065+
if (this == o) {
1066+
return true;
1067+
}
1068+
if (!(o instanceof DateStatisticsImpl)) {
1069+
return false;
1070+
}
1071+
if (!super.equals(o)) {
1072+
return false;
1073+
}
1074+
1075+
DateStatisticsImpl that = (DateStatisticsImpl) o;
1076+
1077+
if (minimum != null ? !minimum.equals(that.minimum) : that.minimum != null) {
1078+
return false;
1079+
}
1080+
if (maximum != null ? !maximum.equals(that.maximum) : that.maximum != null) {
1081+
return false;
1082+
}
1083+
if (minDate != null ? !minDate.equals(that.minDate) : that.minDate != null) {
1084+
return false;
1085+
}
1086+
if (maxDate != null ? !maxDate.equals(that.maxDate) : that.maxDate != null) {
1087+
return false;
1088+
}
1089+
1090+
return true;
1091+
}
1092+
1093+
@Override
1094+
public int hashCode() {
1095+
int result = super.hashCode();
1096+
result = 31 * result + (minimum != null ? minimum.hashCode() : 0);
1097+
result = 31 * result + (maximum != null ? maximum.hashCode() : 0);
1098+
result = 31 * result + (minDate != null ? minDate.hashCode() : 0);
1099+
result = 31 * result + (maxDate != null ? maxDate.hashCode() : 0);
1100+
return result;
1101+
}
8181102
}
8191103

8201104
private static final class TimestampStatisticsImpl extends ColumnStatisticsImpl
@@ -925,6 +1209,38 @@ public String toString() {
9251209
}
9261210
return buf.toString();
9271211
}
1212+
1213+
@Override
1214+
public boolean equals(Object o) {
1215+
if (this == o) {
1216+
return true;
1217+
}
1218+
if (!(o instanceof TimestampStatisticsImpl)) {
1219+
return false;
1220+
}
1221+
if (!super.equals(o)) {
1222+
return false;
1223+
}
1224+
1225+
TimestampStatisticsImpl that = (TimestampStatisticsImpl) o;
1226+
1227+
if (minimum != null ? !minimum.equals(that.minimum) : that.minimum != null) {
1228+
return false;
1229+
}
1230+
if (maximum != null ? !maximum.equals(that.maximum) : that.maximum != null) {
1231+
return false;
1232+
}
1233+
1234+
return true;
1235+
}
1236+
1237+
@Override
1238+
public int hashCode() {
1239+
int result = super.hashCode();
1240+
result = 31 * result + (minimum != null ? minimum.hashCode() : 0);
1241+
result = 31 * result + (maximum != null ? maximum.hashCode() : 0);
1242+
return result;
1243+
}
9281244
}
9291245

9301246
private long count = 0;

0 commit comments

Comments
 (0)