lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(hudi-query): 修复写hdfs文件内容取值错误

Browse Source
This commit is contained in:
v-zhangjc9
2024-05-08 14:14:19 +08:00
parent b2e3a58d27
commit 10fae04007
3 changed files with 52 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.idea/httpRequests/http-requests-log.http generated
View File

@@ -1,3 +1,31 @@
POST http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s8.hdp.dc:15391/hdfs/write?root=hdfs://b2/apps/datalake/test/test.txt&overwrite=true
Content-Type: text/plain
Content-Length: 738
Connection: Keep-Alive
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
Cookie: JSESSIONID=E6FF5447C8553BA4268979B8C5779363
Accept-Encoding: br,deflate,gzip,x-gzip
\#Properties saved on 2023-12-26T09:18:39.583Z
\#Tue Dec 26 17:18:39 CST 2023
hoodie.table.precombine.field=update_ts
hoodie.datasource.write.drop.partition.columns=false
hoodie.table.partition.fields=CITY_ID
hoodie.table.type=MERGE_ON_READ
hoodie.archivelog.folder=archived
hoodie.compaction.payload.class=org.apache.hudi.common.model.OverwriteWithLatestAvroPayload
hoodie.timeline.layout.version=1
hoodie.table.version=4
hoodie.table.recordkey.fields=_key
hoodie.datasource.write.partitionpath.urlencode=false
hoodie.table.name=dws_account
hoodie.table.keygenerator.class=org.apache.hudi.keygen.SimpleKeyGenerator
hoodie.table.timeline.timezone=LOCAL
hoodie.datasource.write.hive_style_partitioning=false
hoodie.table.checksum=989688289
###
POST http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s8.hdp.dc:15391/hdfs/write?root=hdfs://b2/apps/datalake/test/test.txt&overwrite=true POST http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s8.hdp.dc:15391/hdfs/write?root=hdfs://b2/apps/datalake/test/test.txt&overwrite=true
Content-Length: 11 Content-Length: 11
Content-Type: */*; charset=UTF-8 Content-Type: */*; charset=UTF-8
@@ -426,11 +454,3 @@ Accept-Encoding: br,deflate,gzip,x-gzip
### ###
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre
Connection: Keep-Alive
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.9)
Cookie: JSESSIONID=9AB8D98C10FACE15EA1CB758D79F8877
Accept-Encoding: br,deflate,gzip,x-gzip
###

10
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/HudiService.java
View File

@@ -1,6 +1,8 @@
package com.lanyuanxiaoyao.service.forest.service; package com.lanyuanxiaoyao.service.forest.service;
import com.dtflys.forest.annotation.BaseRequest; import com.dtflys.forest.annotation.BaseRequest;
import com.dtflys.forest.annotation.Body;
import com.dtflys.forest.annotation.BodyType;
import com.dtflys.forest.annotation.Get; import com.dtflys.forest.annotation.Get;
import com.dtflys.forest.annotation.Post; import com.dtflys.forest.annotation.Post;
import com.dtflys.forest.annotation.Query; import com.dtflys.forest.annotation.Query;
@@ -77,11 +79,11 @@ public interface HudiService {
@Get("/hdfs/read") @Get("/hdfs/read")
String read(@Query("root") String root); String read(@Query("root") String root);
@Post("/hdfs/write") @Post(value = "/hdfs/write", contentType = "plain/text")
String write(@Query("root") String root, String text); String write(@Query("root") String root, @Body String text);
@Post("/hdfs/write") @Post(value = "/hdfs/write", contentType = "plain/text")
String write(@Query("root") String root, String text, @Query("overwrite") Boolean overwrite); String write(@Query("root") String root, @Body String text, @Query("overwrite") Boolean overwrite);
@Get("/hdfs/download") @Get("/hdfs/download")
InputStream download(@Query("root") String root); InputStream download(@Query("root") String root);

19
test/test.http
View File

@@ -102,5 +102,22 @@ GET http://{{username}}:{{password}}@b12s10.hdp.dc:16695/hdfs/list?root=hdfs://b
### Test HDFS write ### Test HDFS write
POST http://{{username}}:{{password}}@b12s8.hdp.dc:15391/hdfs/write?root=hdfs://b2/apps/datalake/test/test.txt&overwrite=true POST http://{{username}}:{{password}}@b12s8.hdp.dc:15391/hdfs/write?root=hdfs://b2/apps/datalake/test/test.txt&overwrite=true
Content-Type: text/plain
Hello world \#Properties saved on 2023-12-26T09:18:39.583Z
\#Tue Dec 26 17:18:39 CST 2023
hoodie.table.precombine.field=update_ts
hoodie.datasource.write.drop.partition.columns=false
hoodie.table.partition.fields=CITY_ID
hoodie.table.type=MERGE_ON_READ
hoodie.archivelog.folder=archived
hoodie.compaction.payload.class=org.apache.hudi.common.model.OverwriteWithLatestAvroPayload
hoodie.timeline.layout.version=1
hoodie.table.version=4
hoodie.table.recordkey.fields=_key
hoodie.datasource.write.partitionpath.urlencode=false
hoodie.table.name=dws_account
hoodie.table.keygenerator.class=org.apache.hudi.keygen.SimpleKeyGenerator
hoodie.table.timeline.timezone=LOCAL
hoodie.datasource.write.hive_style_partitioning=false
hoodie.table.checksum=989688289