lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3670] free temp views in sql transformers (#5080)

Browse Source
This commit is contained in:
Qi Ji
2022-06-01 22:35:40 +08:00
committed by GitHub
parent dfcd6d9a86
commit 7276d0eaa6
3 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/FlatteningTransformer.java
View File

@@ -49,8 +49,10 @@ public class FlatteningTransformer implements Transformer {
// tmp table name doesn't like dashes // tmp table name doesn't like dashes
String tmpTable = TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_")); String tmpTable = TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_"));
LOG.info("Registering tmp table : " + tmpTable); LOG.info("Registering tmp table : " + tmpTable);
rowDataset.registerTempTable(tmpTable); rowDataset.createOrReplaceTempView(tmpTable);
return sparkSession.sql("select " + flattenSchema(rowDataset.schema(), null) + " from " + tmpTable); Dataset<Row> transformed = sparkSession.sql("select " + flattenSchema(rowDataset.schema(), null) + " from " + tmpTable);
sparkSession.catalog().dropTempView(tmpTable);
return transformed;
} }
public String flattenSchema(StructType schema, String prefix) { public String flattenSchema(StructType schema, String prefix) {

4
hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlFileBasedTransformer.java
View File

@@ -76,7 +76,7 @@ public class SqlFileBasedTransformer implements Transformer {
// tmp table name doesn't like dashes // tmp table name doesn't like dashes
final String tmpTable = TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_")); final String tmpTable = TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_"));
LOG.info("Registering tmp table : " + tmpTable); LOG.info("Registering tmp table : " + tmpTable);
rowDataset.registerTempTable(tmpTable); rowDataset.createOrReplaceTempView(tmpTable);
try (final Scanner scanner = new Scanner(fs.open(new Path(sqlFile)), "UTF-8")) { try (final Scanner scanner = new Scanner(fs.open(new Path(sqlFile)), "UTF-8")) {
Dataset<Row> rows = null; Dataset<Row> rows = null;
@@ -95,6 +95,8 @@ public class SqlFileBasedTransformer implements Transformer {
return rows; return rows;
} catch (final IOException ioe) { } catch (final IOException ioe) {
throw new HoodieIOException("Error reading transformer SQL file.", ioe); throw new HoodieIOException("Error reading transformer SQL file.", ioe);
} finally {
sparkSession.catalog().dropTempView(tmpTable);
} }
} }

6
hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java
View File

@@ -60,9 +60,11 @@ public class SqlQueryBasedTransformer implements Transformer {
// tmp table name doesn't like dashes // tmp table name doesn't like dashes
String tmpTable = TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_")); String tmpTable = TMP_TABLE.concat(UUID.randomUUID().toString().replace("-", "_"));
LOG.info("Registering tmp table : " + tmpTable); LOG.info("Registering tmp table : " + tmpTable);
rowDataset.registerTempTable(tmpTable); rowDataset.createOrReplaceTempView(tmpTable);
String sqlStr = transformerSQL.replaceAll(SRC_PATTERN, tmpTable); String sqlStr = transformerSQL.replaceAll(SRC_PATTERN, tmpTable);
LOG.debug("SQL Query for transformation : (" + sqlStr + ")"); LOG.debug("SQL Query for transformation : (" + sqlStr + ")");
return sparkSession.sql(sqlStr); Dataset<Row> transformed = sparkSession.sql(sqlStr);
sparkSession.catalog().dropTempView(tmpTable);
return transformed;
} }
} }