lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(executor-task): 增加一个mini hudi的功能

Browse Source 
从pulsar到hdfs落盘
This commit is contained in:
v-zhangjc9
2024-05-28 18:16:09 +08:00
parent 095347cd1a
commit deae4fd294
11 changed files with 507 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
test/test.http
View File

@@ -47,10 +47,12 @@ Content-Type: application/json
GET {{queue-url}}/queue/clear?name=compaction-queue-pre
### Info
GET http://{{username}}:{{password}}@132.122.116.146:18166/info/compaction_metrics?flink_job_id=1542097996099055616&alias=acct_acct_item_fs&filter_completes=true
GET http://{{username}}:{{password}}@132.122.116.146:18166/info/compaction_metrics?flink_job_id=1542097996099055616&
alias=acct_acct_item_fs&filter_completes=true
### Info
GET http://{{username}}:{{password}}@132.122.116.150:27510/table/list_compaction_metrics?search_flink_job_id=1542097996099055616&search_alias=acct_acct_item_fs
GET http://{{username}}:{{password}}@132.122.116.150:27510/table/list_compaction_metrics?
search_flink_job_id=1542097996099055616&search_alias=acct_acct_item_fs
### Query Scheduler
GET {{scheduler-url}}/schedule/all
@@ -86,7 +88,8 @@ GET {{web-url}}/overview/sync_running_status
GET {{scheduler-url}}/schedule/schedule_times
### Get message id
GET {{api-url}}/api/sync_checkpoint_state?flink_job_id=1542097996099055616&alias=acct_acct_item_zs&message_id=861976:46933:-1&publish_time=1705373846898
GET {{api-url}}/api/sync_checkpoint_state?flink_job_id=1542097996099055616&alias=acct_acct_item_zs&
message_id=861976:46933:-1&publish_time=1705373846898
### Test
GET {{web-url}}/test
@@ -95,13 +98,17 @@ GET {{web-url}}/test
GET {{exporter-url}}/exporter/un_running_flink_job
### Pulsar backlog
GET http://{{username}}:{{password}}@b12s15.hdp.dc:21685/pulsar/backlog?name=main&topic=persistent://odcp/grid/grid_serv_staff&subscription=Hudi_Sync_Pulsar_Reader_1552408245762723840_grid_grid_serv_staff_b_20230425
GET http://{{username}}:{{password}}@b12s15.hdp.dc:21685/pulsar/backlog?name=main&
topic=persistent://odcp/grid/grid_serv_staff&
subscription=Hudi_Sync_Pulsar_Reader_1552408245762723840_grid_grid_serv_staff_b_20230425
### Test HDFS list
GET http://{{username}}:{{password}}@b12s10.hdp.dc:16695/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr
GET http://{{username}}:{{password}}@b12s10.hdp.dc:16695/hdfs/list?
root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr
### Test HDFS write
POST http://{{username}}:{{password}}@b12s8.hdp.dc:15391/hdfs/write?root=hdfs://b2/apps/datalake/test/test.txt&overwrite=true
POST http://{{username}}:{{password}}@b12s8.hdp.dc:15391/hdfs/write?root=hdfs://b2/apps/datalake/test/test.txt&
overwrite=true
Content-Type: text/plain
\#Properties saved on 2023-12-26T09:18:39.583Z
@@ -121,3 +128,9 @@ hoodie.table.keygenerator.class=org.apache.hudi.keygen.SimpleKeyGenerator
hoodie.table.timeline.timezone=LOCAL
hoodie.datasource.write.hive_style_partitioning=false
hoodie.table.checksum=989688289
### Test police
GET http://{{username}}:{{password}}@b12s10.hdp.dc:33535/task/law_enforcement?
pulsar_url=pulsar://132.122.115.158:16650,132.122.115.159:16650,132.122.115.160:16650,132.122.115.161:16650,132.122.115.167:16650,132.122.115.168:16650&
pulsar_topic=persistent://odcp/acct_dg/acct_item_760&start_time=1716858000000&end_time=1716861600000&
primary_keys=ACCT_ITEM_ID&partition_keys=ACCT_ID