ORC-120. Add option to force positional matching of schema evolution.

omalley · omalley · commit db00ffdb9f8a · 2016-12-14T08:25:13.000-08:00
Fixes apache#72 Signed-off-by: Owen O'Malley <omalley@apache.org>
diff --git a/java/core/src/java/org/apache/orc/OrcConf.java b/java/core/src/java/org/apache/orc/OrcConf.java
@@ -137,7 +137,12 @@ public enum OrcConf {
   KRYO_SARG("orc.kryo.sarg", "orc.kryo.sarg", null,
       "The kryo and base64 encoded SearchArgument for predicate pushdown."),
   SARG_COLUMNS("orc.sarg.column.names", "org.sarg.column.names", null,
-      "The list of column names for the SearchArgument.")
+      "The list of column names for the SearchArgument."),
+  FORCE_POSITIONAL_EVOLUTION("orc.force.positional.evolution",
+      "orc.force.positional.evolution", false,
+      "Require schema evolution to match the top level columns using position\n" +
+      "rather than column names. This provides backwards compatibility with\n" +
+      "Hive 2.1.")
   ;
 
   private final String attribute;
diff --git a/java/core/src/java/org/apache/orc/Reader.java b/java/core/src/java/org/apache/orc/Reader.java
@@ -159,6 +159,7 @@ public static class Options {
     private TypeDescription schema = null;
     private DataReader dataReader = null;
     private Boolean tolerateMissingSchema = null;
+    private boolean forcePositionalEvolution;
 
     public Options() {
       // PASS
@@ -168,6 +169,7 @@ public Options(Configuration conf) {
       useZeroCopy = OrcConf.USE_ZEROCOPY.getBoolean(conf);
       skipCorruptRecords = OrcConf.SKIP_CORRUPT_DATA.getBoolean(conf);
       tolerateMissingSchema = OrcConf.TOLERATE_MISSING_SCHEMA.getBoolean(conf);
+      forcePositionalEvolution = OrcConf.FORCE_POSITIONAL_EVOLUTION.getBoolean(conf);
     }
 
     /**
@@ -249,6 +251,17 @@ public Options tolerateMissingSchema(boolean value) {
       return this;
     }
 
+    /**
+     * Set whether to force schema evolution to be positional instead of
+     * based on the column names.
+     * @param value force positional evolution
+     * @return this
+     */
+    public Options forcePositionalEvolution(boolean value) {
+      this.forcePositionalEvolution = value;
+      return this;
+    }
+
     public boolean[] getInclude() {
       return include;
     }
@@ -293,6 +306,10 @@ public DataReader getDataReader() {
       return dataReader;
     }
 
+    public boolean getForcePositionalEvolution() {
+      return forcePositionalEvolution;
+    }
+
     public Options clone() {
       Options result = new Options();
       result.include = include;
diff --git a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
@@ -95,7 +95,10 @@ public SchemaEvolution(TypeDescription fileSchema,
       this.readerFileTypes =
         new TypeDescription[this.readerSchema.getMaximumId() + 1];
       int positionalLevels = 0;
-      if (!hasColumnNames(isAcid? getBaseRow(fileSchema) : fileSchema)){
+      if (options.getForcePositionalEvolution()) {
+        positionalLevels = isAcid ? 2 : 1;
+        buildConversion(fileSchema, this.readerSchema, positionalLevels);
+      } else if (!hasColumnNames(isAcid? getBaseRow(fileSchema) : fileSchema)) {
         if (!this.fileSchema.equals(this.readerSchema)) {
           if (!allowMissingMetadata) {
             throw new RuntimeException("Found that schema metadata is missing"
diff --git a/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java b/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java
@@ -1073,4 +1073,16 @@ public void testTypeConversion() throws IOException {
           ((LongColumnVector) batch.cols[2]).vector[r]);
     }
   }
+
+  @Test
+  public void testPositionalEvolution() throws IOException {
+    options.forcePositionalEvolution(true);
+    TypeDescription file = TypeDescription.fromString("struct<x:int,y:int,z:int>");
+    TypeDescription read = TypeDescription.fromString("struct<z:int,x:int,a:int,b:int>");
+    SchemaEvolution evo = new SchemaEvolution(file, read, options);
+    assertEquals(1, evo.getFileType(1).getId());
+    assertEquals(2, evo.getFileType(2).getId());
+    assertEquals(3, evo.getFileType(3).getId());
+    assertEquals(null, evo.getFileType(4));
+  }
 }

Original file line number	Diff line number	Diff line change
`@@ -1073,4 +1073,16 @@ public void testTypeConversion() throws IOException {`
`1073`	`1073`	`((LongColumnVector) batch.cols[2]).vector[r]);`
`1074`	`1074`	`}`
`1075`	`1075`	`}`
	`1076`	`+`
	`1077`	`+ @Test`
	`1078`	`+ public void testPositionalEvolution() throws IOException {`
	`1079`	`+ options.forcePositionalEvolution(true);`
	`1080`	`+ TypeDescription file = TypeDescription.fromString("struct<x:int,y:int,z:int>");`
	`1081`	`+ TypeDescription read = TypeDescription.fromString("struct<z:int,x:int,a:int,b:int>");`
	`1082`	`+ SchemaEvolution evo = new SchemaEvolution(file, read, options);`
	`1083`	`+ assertEquals(1, evo.getFileType(1).getId());`
	`1084`	`+ assertEquals(2, evo.getFileType(2).getId());`
	`1085`	`+ assertEquals(3, evo.getFileType(3).getId());`
	`1086`	`+ assertEquals(null, evo.getFileType(4));`
	`1087`	`+ }`
`1076`	`1088`	`}`