From 6e2443468280dfd768afe9ac4d17df7fbbbb51bf Mon Sep 17 00:00:00 2001 From: Shawy Geng Date: Tue, 6 Jul 2021 15:35:12 +0800 Subject: [PATCH] [HUDI-2113] Fix integration testing failure caused by sql results out of order (#3204) --- docker/demo/hive-batch1.commands | 12 ++++++------ docker/demo/hive-batch2-after-compaction.commands | 8 ++++---- .../java/org/apache/hudi/integ/ITTestHoodieDemo.java | 12 ++++++++++++ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/docker/demo/hive-batch1.commands b/docker/demo/hive-batch1.commands index 021c6d55b..ed2eaca8a 100644 --- a/docker/demo/hive-batch1.commands +++ b/docker/demo/hive-batch1.commands @@ -21,16 +21,16 @@ select symbol, max(ts) from stock_ticks_cow group by symbol HAVING symbol = 'GOO select symbol, max(ts) from stock_ticks_mor_ro group by symbol HAVING symbol = 'GOOG'; select symbol, max(ts) from stock_ticks_mor_rt group by symbol HAVING symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_cow where symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_mor_ro where symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_mor_rt where symbol = 'GOOG'; +select symbol, ts, volume, open, close from stock_ticks_cow where symbol = 'GOOG' order by ts; +select symbol, ts, volume, open, close from stock_ticks_mor_ro where symbol = 'GOOG' order by ts; +select symbol, ts, volume, open, close from stock_ticks_mor_rt where symbol = 'GOOG' order by ts; select symbol, max(ts) from stock_ticks_cow_bs group by symbol HAVING symbol = 'GOOG'; select symbol, max(ts) from stock_ticks_mor_bs_ro group by symbol HAVING symbol = 'GOOG'; select symbol, max(ts) from stock_ticks_mor_bs_rt group by symbol HAVING symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_cow_bs where symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_mor_bs_ro where symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_mor_bs_rt where symbol = 'GOOG'; +select symbol, ts, volume, open, close from stock_ticks_cow_bs where symbol = 'GOOG' order by ts; +select symbol, ts, volume, open, close from stock_ticks_mor_bs_ro where symbol = 'GOOG' order by ts; +select symbol, ts, volume, open, close from stock_ticks_mor_bs_rt where symbol = 'GOOG' order by ts; !quit diff --git a/docker/demo/hive-batch2-after-compaction.commands b/docker/demo/hive-batch2-after-compaction.commands index 06582a309..9d5a0b27d 100644 --- a/docker/demo/hive-batch2-after-compaction.commands +++ b/docker/demo/hive-batch2-after-compaction.commands @@ -20,13 +20,13 @@ add jar ${hudi.hadoop.bundle}; select symbol, max(ts) from stock_ticks_mor_ro group by symbol HAVING symbol = 'GOOG'; select symbol, max(ts) from stock_ticks_mor_rt group by symbol HAVING symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_mor_ro where symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_mor_rt where symbol = 'GOOG'; +select symbol, ts, volume, open, close from stock_ticks_mor_ro where symbol = 'GOOG' order by ts; +select symbol, ts, volume, open, close from stock_ticks_mor_rt where symbol = 'GOOG' order by ts; select symbol, max(ts) from stock_ticks_mor_bs_ro group by symbol HAVING symbol = 'GOOG'; select symbol, max(ts) from stock_ticks_mor_bs_rt group by symbol HAVING symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_mor_bs_ro where symbol = 'GOOG'; -select symbol, ts, volume, open, close from stock_ticks_mor_bs_rt where symbol = 'GOOG'; +select symbol, ts, volume, open, close from stock_ticks_mor_bs_ro where symbol = 'GOOG' order by ts; +select symbol, ts, volume, open, close from stock_ticks_mor_bs_rt where symbol = 'GOOG' order by ts; !quit diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java index 0fa9b0ffa..c8de880bd 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java @@ -24,6 +24,7 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.keygen.SimpleKeyGenerator; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -89,6 +90,17 @@ public class ITTestHoodieDemo extends ITTestBase { + " --hoodie-conf hoodie.datasource.hive_sync.database=default " + " --hoodie-conf hoodie.datasource.hive_sync.table=%s"; + + @AfterEach + public void clean() throws Exception { + String hdfsCmd = "hdfs dfs -rm -R "; + List tablePaths = CollectionUtils.createImmutableList( + COW_BASE_PATH, MOR_BASE_PATH, COW_BOOTSTRAPPED_BASE_PATH, MOR_BOOTSTRAPPED_BASE_PATH); + for (String tablePath : tablePaths) { + executeCommandStringInDocker(ADHOC_1_CONTAINER, hdfsCmd + tablePath, true); + } + } + @Test public void testParquetDemo() throws Exception { baseFileFormat = HoodieFileFormat.PARQUET;