lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

synchronized lock on conf object instead of class

Browse Source
This commit is contained in:
Wenning Ding
2019-10-30 11:48:21 -07:00
committed by n3nash
parent 3251d62bd3
commit ee0fd06de7
1 changed files with 10 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java
View File

@@ -200,9 +200,11 @@ public class HoodieParquetRealtimeInputFormat extends HoodieParquetInputFormat i
/** /**
* Hive will append read columns' ids to old columns' ids during getRecordReader. In some cases, e.g. SELECT COUNT(*), * Hive will append read columns' ids to old columns' ids during getRecordReader. In some cases, e.g. SELECT COUNT(*),
* the read columns' id is an empty string and Hive will combine it with Hoodie required projection ids and becomes * the read columns' id is an empty string and Hive will combine it with Hoodie required projection ids and becomes
* e.g. ",2,0,3" and will cause an error. This method is used to avoid this situation. * e.g. ",2,0,3" and will cause an error. Actually this method is a temporary solution because the real bug is from
* Hive. Hive has fixed this bug after 3.0.0, but the version before that would still face this problem. (HIVE-22438)
*/ */
private static synchronized Configuration cleanProjectionColumnIds(Configuration conf) { private static Configuration cleanProjectionColumnIds(Configuration conf) {
synchronized (conf) {
String columnIds = conf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR); String columnIds = conf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR);
if (!columnIds.isEmpty() && columnIds.charAt(0) == ',') { if (!columnIds.isEmpty() && columnIds.charAt(0) == ',') {
conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, columnIds.substring(1)); conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, columnIds.substring(1));
@@ -210,6 +212,7 @@ public class HoodieParquetRealtimeInputFormat extends HoodieParquetInputFormat i
LOG.debug("The projection Ids: {" + columnIds + "} start with ','. First comma is removed"); LOG.debug("The projection Ids: {" + columnIds + "} start with ','. First comma is removed");
} }
} }
}
return conf; return conf;
} }