lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Final cleanup before open-source

Browse Source
This commit is contained in:
Prasanna Rajaperumal
2016-12-16 14:43:39 -08:00
parent 81874a8406
commit 8e80c8d2ea
3 changed files with 10 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hoodie-cli/pom.xml
View File

@@ -193,6 +193,16 @@
<version>${junit.version}</version> <version>${junit.version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.9.6</version>
</dependency>
</dependencies> </dependencies>
</project> </project>

2
hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/HiveUtil.java
View File

@@ -17,7 +17,6 @@
package com.uber.hoodie.cli.utils; package com.uber.hoodie.cli.utils;
import com.uber.hoodie.common.model.HoodieTableMetadata; import com.uber.hoodie.common.model.HoodieTableMetadata;
import com.uber.hoodie.hadoop.HoodieInputFormat;
import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.dbcp.BasicDataSource;
import org.joda.time.DateTime; import org.joda.time.DateTime;
@@ -62,7 +61,6 @@ public class HiveUtil {
//stmt.execute("set mapred.job.queue.name=<queue_name>"); //stmt.execute("set mapred.job.queue.name=<queue_name>");
stmt.execute("set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat" ); stmt.execute("set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat" );
stmt.execute("set hive.stats.autogather=false" ); stmt.execute("set hive.stats.autogather=false" );
System.out.println("Class " + HoodieInputFormat.class.getName());
rs = stmt.executeQuery( rs = stmt.executeQuery(
"select count(`_hoodie_commit_time`) as cnt from " + dbName + "." + source "select count(`_hoodie_commit_time`) as cnt from " + dbName + "." + source
.getTableName()); .getTableName());

20
hoodie-common/src/main/java/com/uber/hoodie/common/GenericHoodiePayload.java
View File

@@ -21,15 +21,11 @@ import com.uber.hoodie.common.model.HoodieRecordPayload;
import org.apache.avro.Schema; import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord; import org.apache.avro.generic.IndexedRecord;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.StringWriter; import java.io.StringWriter;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.zip.Deflater; import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream; import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream; import java.util.zip.InflaterInputStream;
@@ -56,22 +52,6 @@ public class GenericHoodiePayload implements HoodieRecordPayload<GenericHoodiePa
return jsonConverter.convert(getJsonData()); return jsonConverter.convert(getJsonData());
} }
public String getRowKey(HoodieSQLStreamer.Config cfg) throws IOException, JSONException {
JSONObject object = new JSONObject(getJsonData());
if(!object.has(cfg.keyColumnField)) {
return "";
}
return (String) object.get(cfg.keyColumnField);
}
public String getPartitionPath(HoodieSQLStreamer.Config cfg) throws IOException, JSONException {
JSONObject object = new JSONObject(getJsonData());
if(!object.has(cfg.partitionPathField)) {
return "0000/00/00";
}
return object.getString(cfg.partitionPathField);
}
private String getJsonData() throws IOException { private String getJsonData() throws IOException {
return unCompressData(jsonDataCompressed); return unCompressData(jsonDataCompressed);
} }