lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

synchronized lock on conf object instead of class

Browse Source
This commit is contained in:
Wenning Ding
2019-10-30 11:48:21 -07:00
committed by n3nash
parent 3251d62bd3
commit ee0fd06de7
1 changed files with 10 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java
View File

@@ -200,14 +200,17 @@ public class HoodieParquetRealtimeInputFormat extends HoodieParquetInputFormat i
/** /**
* Hive will append read columns' ids to old columns' ids during getRecordReader. In some cases, e.g. SELECT COUNT(*), * Hive will append read columns' ids to old columns' ids during getRecordReader. In some cases, e.g. SELECT COUNT(*),
* the read columns' id is an empty string and Hive will combine it with Hoodie required projection ids and becomes * the read columns' id is an empty string and Hive will combine it with Hoodie required projection ids and becomes
* e.g. ",2,0,3" and will cause an error. This method is used to avoid this situation. * e.g. ",2,0,3" and will cause an error. Actually this method is a temporary solution because the real bug is from
* Hive. Hive has fixed this bug after 3.0.0, but the version before that would still face this problem. (HIVE-22438)
*/ */
private static synchronized Configuration cleanProjectionColumnIds(Configuration conf) { private static Configuration cleanProjectionColumnIds(Configuration conf) {
String columnIds = conf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR); synchronized (conf) {
if (!columnIds.isEmpty() && columnIds.charAt(0) == ',') { String columnIds = conf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR);
conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, columnIds.substring(1)); if (!columnIds.isEmpty() && columnIds.charAt(0) == ',') {
if (LOG.isDebugEnabled()) { conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, columnIds.substring(1));
LOG.debug("The projection Ids: {" + columnIds + "} start with ','. First comma is removed"); if (LOG.isDebugEnabled()) {
LOG.debug("The projection Ids: {" + columnIds + "} start with ','. First comma is removed");
}
} }
} }
return conf; return conf;