feat(monitor): 增加关于hudi表文件数的监控指标

.idea/httpRequests/http-requests-log.http

@@ -1,3 +1,168 @@
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s20.hdp.dc:19521/hdfs/count?root=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+content-length: 0
+
+<> 2024-10-12T154854.500.txt
+
+###
+
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s12.hdp.dc:25961/hdfs/count?root=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+Cookie: JSESSIONID=EE0952421D19909D0A80BB5A1216DE93
+content-length: 0
+
+<> 2024-10-12T154825.500.txt
+
+###
+
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s12.hdp.dc:25961/hdfs/count?root=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+Cookie: JSESSIONID=EE0952421D19909D0A80BB5A1216DE93
+content-length: 0
+
+<> 2024-10-12T154754.500.txt
+
+###
+
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s12.hdp.dc:25961/hdfs/count?root=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+Cookie: JSESSIONID=EE0952421D19909D0A80BB5A1216DE93
+content-length: 0
+
+<> 2024-10-12T154616.500.txt
+
+###
+
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s12.hdp.dc:25961/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+Cookie: JSESSIONID=EE0952421D19909D0A80BB5A1216DE93
+content-length: 0
+
+<> 2024-10-12T154529.500.txt
+
+###
+
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s12.hdp.dc:25961/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+Cookie: JSESSIONID=EE0952421D19909D0A80BB5A1216DE93
+content-length: 0
+
+<> 2024-10-12T151839.500.txt
+
+###
+
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s12.hdp.dc:25961/hdfs/file_count?root=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+Cookie: JSESSIONID=EE0952421D19909D0A80BB5A1216DE93
+content-length: 0
+
+<> 2024-10-12T151753.500.txt
+
+###
+
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s12.hdp.dc:25961/hdfs/file_count?root=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+Cookie: JSESSIONID=EE0952421D19909D0A80BB5A1216DE93
+content-length: 0
+
+<> 2024-10-12T151727.500.txt
+
+###
+
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s12.hdp.dc:25961/hdfs/count?root=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+content-length: 0
+
+<> 2024-10-12T151704.500.txt
+
+###
+
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s22.hdp.dc:13241/hdfs/count?root=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+Cookie: JSESSIONID=D4B48AD7708DF28D7AFA0A74B26CF45A
+content-length: 0
+
+<> 2024-10-12T151540.500.txt
+
+###
+
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s22.hdp.dc:13241/hdfs/count?root=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+Cookie: JSESSIONID=D4B48AD7708DF28D7AFA0A74B26CF45A
+content-length: 0
+
+<> 2024-10-12T151442.500.txt
+
+###
+
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s22.hdp.dc:13241/hdfs/count?root=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+Cookie: JSESSIONID=D4B48AD7708DF28D7AFA0A74B26CF45A
+content-length: 0
+
+<> 2024-10-12T151417.500.txt
+
+###
+
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s22.hdp.dc:13241/hdfs/count?hdfs=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+Cookie: JSESSIONID=D4B48AD7708DF28D7AFA0A74B26CF45A
+content-length: 0
+
+<> 2024-10-12T151409.500.txt
+
+###
+
+GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s22.hdp.dc:13241/hdfs?hdfs=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data
+Authorization: Basic AxhEbscwsJDbYMH2 cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4
+User-Agent: IntelliJ HTTP Client/IntelliJ IDEA 2024.1.4
+Accept-Encoding: br, deflate, gzip, x-gzip
+Accept: */*
+content-length: 0
+
+<> 2024-10-12T151340.404.json
+
+###
+
 GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:31719/task/law_enforcement?pulsar_url=pulsar://132.122.115.158:16650,132.122.115.159:16650,132.122.115.160:16650,132.122.115.161:16650,132.122.115.167:16650,132.122.115.168:16650&pulsar_topic=persistent://odcp/acct_sz/acct_item_755&start_time=1716858000000&end_time=1716861600000&primary_keys=ACCT_ITEM_ID&partition_keys=ACCT_ID
 Connection: Keep-Alive
 User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
@@ -358,163 +523,3 @@ Accept-Encoding: br,deflate,gzip,x-gzip
 
 ###
 
-GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:33535/task/law_enforcement?pulsar_url=pulsar://132.122.115.158:16650,132.122.115.159:16650,132.122.115.160:16650,132.122.115.161:16650,132.122.115.167:16650,132.122.115.168:16650&pulsar_topic=persistent://odcp/acct_dg/acct_item_760&start_time=1716858000000&end_time=1716861600000&primary_keys=ACCT_ITEM_ID&partition_keys=ACCT_ID
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=08E78CE8926806AAB5D110D0FE9B05F7
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-<> 2024-05-28T164901.200.txt
-
-###
-
-GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:33535/task/law_enforcement?pulsar_url=pulsar://132.122.115.158:16650,132.122.115.159:16650,132.122.115.160:16650,132.122.115.161:16650,132.122.115.167:16650,132.122.115.168:16650&pulsar_topic=persistent://odcp/acct_dg/acct_item_760&start_time=1716858000000&end_time=1716861600000&primary_keys=ACCT_ITEM_ID&partition_keys=ACCT_ID
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=08E78CE8926806AAB5D110D0FE9B05F7
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-<> 2024-05-28T164758.200.txt
-
-###
-
-GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:33535/task/law_enforcement?pulsar_url=pulsar://132.122.115.158:16650,132.122.115.159:16650,132.122.115.160:16650,132.122.115.161:16650,132.122.115.167:16650,132.122.115.168:16650&pulsar_topic=persistent://odcp/acct_dg/acct_item_760&start_time=1716858000000&end_time=1716861600000&primary_keys=ACCT_ITEM_ID&partition_keys=ACCT_ID
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=08E78CE8926806AAB5D110D0FE9B05F7
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-<> 2024-05-28T164303.200.txt
-
-###
-
-GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:33535/task/law_enforcement?pulsar_url=pulsar://132.122.115.158:16650,132.122.115.159:16650,132.122.115.160:16650,132.122.115.161:16650,132.122.115.167:16650,132.122.115.168:16650&pulsar_topic=persistent://odcp/acct_dg/acct_item_760&start_time=1716858000000&end_time=1716861600000&primary_keys=ACCT_ITEM_ID&partition_keys=ACCT_ID
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=08E78CE8926806AAB5D110D0FE9B05F7
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-<> 2024-05-28T164220.200.txt
-
-###
-
-GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:33535/task/law_enforcement?pulsar_url=pulsar://132.122.115.158:16650,132.122.115.159:16650,132.122.115.160:16650,132.122.115.161:16650,132.122.115.167:16650,132.122.115.168:16650&pulsar_topic=persistent://odcp/acct_dg/acct_item_760&start_time=1716858000000&end_time=1716861600000&primary_keys=ACCT_ITEM_ID&partition_keys=ACCT_ID
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=08E78CE8926806AAB5D110D0FE9B05F7
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-<> 2024-05-28T164107.200.txt
-
-###
-
-GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:33535/task/law_enforcement?pulsar_url=pulsar://132.122.115.158:16650,132.122.115.159:16650,132.122.115.160:16650,132.122.115.161:16650,132.122.115.167:16650,132.122.115.168:16650&pulsar_topic=persistent://odcp/acct_dg/acct_item_760&start_time=1716858000000&end_time=1716861600000&primary_keys=ACCT_ITEM_ID&partition_keys=ACCT_ID
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=C5D2666661F27F68E53223FE5B74AF35
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-<> 2024-05-28T163410.200.txt
-
-###
-
-GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=8516C92140B5118AF9AA61025D0F8C93
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-###
-
-GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/service_scheduler/schedule/all
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=7A4C34E0240A98C1186F3A2551BC5E80
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-###
-
-GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/service_web/cloud/list
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=F5F155198FAF72435339CC2E21B873CC
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-<> 2024-05-09T170723.200.json
-
-###
-
-GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/hudi_api/api/message_id?flink_job_id=1542097984132706304&alias=crm_cfguse_mkt_cam_strategy_rel
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=F5F155198FAF72435339CC2E21B873CC
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-###
-
-POST http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s8.hdp.dc:15391/hdfs/write?root=hdfs://b2/apps/datalake/test/test.txt&overwrite=true
-Content-Type: text/plain
-Content-Length: 738
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=E6FF5447C8553BA4268979B8C5779363
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-\#Properties saved on 2023-12-26T09:18:39.583Z
-\#Tue Dec 26 17:18:39 CST 2023
-hoodie.table.precombine.field=update_ts
-hoodie.datasource.write.drop.partition.columns=false
-hoodie.table.partition.fields=CITY_ID
-hoodie.table.type=MERGE_ON_READ
-hoodie.archivelog.folder=archived
-hoodie.compaction.payload.class=org.apache.hudi.common.model.OverwriteWithLatestAvroPayload
-hoodie.timeline.layout.version=1
-hoodie.table.version=4
-hoodie.table.recordkey.fields=_key
-hoodie.datasource.write.partitionpath.urlencode=false
-hoodie.table.name=dws_account
-hoodie.table.keygenerator.class=org.apache.hudi.keygen.SimpleKeyGenerator
-hoodie.table.timeline.timezone=LOCAL
-hoodie.datasource.write.hive_style_partitioning=false
-hoodie.table.checksum=989688289
-
-###
-
-POST http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s8.hdp.dc:15391/hdfs/write?root=hdfs://b2/apps/datalake/test/test.txt&overwrite=true
-Content-Length: 11
-Content-Type: */*; charset=UTF-8
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=D12E206603C453F1429C0B7DF1519A4B
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-Hello world
-
-###
-
-POST http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s8.hdp.dc:34469/hdfs/write?root=hdfs://b2/apps/datalake/test/test.txt&overwrite=true
-Content-Length: 11
-Content-Type: */*; charset=UTF-8
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=D12E206603C453F1429C0B7DF1519A4B
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-Hello world
-
-###
-
-POST http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s8.hdp.dc:34469/hdfs/write?root=hdfs://b2/apps/datalake/test/test.txt
-Content-Length: 11
-Content-Type: */*; charset=UTF-8
-Connection: Keep-Alive
-User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
-Cookie: JSESSIONID=D12E206603C453F1429C0B7DF1519A4B
-Accept-Encoding: br,deflate,gzip,x-gzip
-
-Hello world
-
-<> 2024-05-08T095641.500.txt
-
-###
-

service-command-pro/src/main/java/com/lanyuanxiaoyao/service/command/pro/commands/HudiCommand.java

@@ -205,6 +205,41 @@ public class HudiCommand {
         fileSystem.close();
     }
 
+    @ShellMethod("Max meta files")
+    public void maxMetaFiles() throws IOException {
+        MutableList<P> list = Lists.mutable.<P>empty().asSynchronized();
+        FileSystem fileSystem = FileSystem.get(new Configuration());
+        infoService
+                .tableMetaList()
+                .collect(TableMeta::getHudi)
+                .collect(TableMeta.HudiMeta::getTargetHdfsPath)
+                .asParallel(ExecutorProvider.EXECUTORS_20, 1)
+                .forEach(hdfs -> {
+                    Path root = new Path(hdfs, ".hoodie");
+                    try {
+                        FileStatus[] statuses = fileSystem.listStatus(root);
+                        long num = 0;
+                        for (FileStatus status : statuses) {
+                            if (status.isFile()) {
+                                num++;
+                            }
+                            if (StrUtil.containsIgnoreCase(status.getPath().toString(), "INVALID")) {
+                                logger.info("{}", status.getPath().toString());
+                            }
+                        }
+                        list.add(new P(num, hdfs));
+                        logger.info("Count: {} Hdfs: {}", num, hdfs);
+                    } catch (IOException e) {
+                        logger.warn("List file error", e);
+                    }
+                });
+        MutableList<P> listP = list.select(p -> p.count > 1000);
+        for (P maxP : listP) {
+            logger.info("Max: {} Hdfs: {}", maxP.count, maxP.hdfs);
+        }
+        fileSystem.close();
+    }
+
     @ShellMethod("Get timeline instants")
     public void timelineInstant(@ShellOption(help = "root hdfs path") String hdfs) {
         hudiService.timelineHdfsAllActive(hdfs).forEach(instant -> logger.info(instant.toString()));
@@ -229,4 +264,14 @@ public class HudiCommand {
     public interface Runnable {
         void run(LongAdder counter);
     }
+
+    private static final class P {
+        String hdfs;
+        long count;
+
+        public P(long count, String hdfs) {
+            this.count = count;
+            this.hdfs = hdfs;
+        }
+    }
 }

service-common/src/main/java/com/lanyuanxiaoyao/service/common/Constants.java

@@ -128,6 +128,12 @@ public interface Constants {
     String METRICS_PULSAR_PREFIX = METRICS_PREFIX + "_pulsar";
     String METRICS_PULSAR_BACKLOG = METRICS_PULSAR_PREFIX + "_backlog";
 
+    String METRICS_HUDI_TABLE = METRICS_PREFIX + "_hudi_table";
+    String METRICS_HUDI_TABLE_FILE_COUNT = METRICS_HUDI_TABLE + "_file_count";
+    String METRICS_HUDI_TABLE_FILE_COUNT_AVERAGE_PER_TABLE = METRICS_HUDI_TABLE_FILE_COUNT + "_average_per_table";
+    String METRICS_HUDI_TABLE_TIMELINE_FILE_COUNT = METRICS_HUDI_TABLE + "_timeline_file_count";
+    String METRICS_HUDI_TABLE_TIMELINE_FILE_COUNT_AVERAGE_PER_TABLE = METRICS_HUDI_TABLE_TIMELINE_FILE_COUNT + "_average_per_table";
+
     String METRICS_LABEL_FLINK_JOB_ID = "flink_job_id";
     String METRICS_LABEL_FLINK_JOB_NAME = "flink_job_name";
     String METRICS_LABEL_FLINK_NATIVE_JOB_ID = "flink_native_job_id";

service-monitor/src/main/java/com/lanyuanxiaoyao/service/monitor/metric/HudiMetrics.java

@@ -0,0 +1,94 @@
+package com.lanyuanxiaoyao.service.monitor.metric;
+
+import cn.hutool.core.util.StrUtil;
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.configuration.ExecutorProvider;
+import com.lanyuanxiaoyao.service.forest.service.HudiService;
+import com.lanyuanxiaoyao.service.forest.service.InfoService;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tag;
+import java.util.concurrent.atomic.AtomicLong;
+import org.eclipse.collections.api.factory.Lists;
+import org.eclipse.collections.api.factory.Maps;
+import org.eclipse.collections.api.map.MutableMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+/**
+ * Hudi表相关指标
+ *
+ * @author lanyuanxiaoyao
+ * @date 2024-03-05
+ */
+@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
+@Service
+public class HudiMetrics extends Metrics {
+    private static final Logger logger = LoggerFactory.getLogger(HudiMetrics.class);
+
+    private final MeterRegistry registry;
+    private final InfoService infoService;
+    private final HudiService hudiService;
+
+    private final MutableMap<String, AtomicLong> fileCountCacheMap;
+    private final MutableMap<String, AtomicLong> timelineFileCountCacheMap;
+
+    public HudiMetrics(MeterRegistry registry, InfoService infoService, HudiService hudiService) {
+        this.registry = registry;
+        this.infoService = infoService;
+        this.hudiService = hudiService;
+
+        fileCountCacheMap = Maps.mutable.empty();
+        timelineFileCountCacheMap = Maps.mutable.empty();
+    }
+
+    @Scheduled(cron = "0 30 * * * ?")
+    @Override
+    public void update() {
+        infoService.tableMetaList()
+                .asParallel(ExecutorProvider.EXECUTORS_10, 1)
+                .reject(meta -> StrUtil.isBlank(meta.getPulsarAddress()))
+                .forEach(meta -> {
+                    try {
+                        AtomicLong filecountCache = fileCountCacheMap.getIfAbsentPut(
+                                meta.getAlias(),
+                                registry.gauge(
+                                        Constants.METRICS_HUDI_TABLE_FILE_COUNT,
+                                        Lists.immutable.of(
+                                                Tag.of(Constants.METRICS_LABEL_FLINK_JOB_ID, meta.getJob().getId().toString()),
+                                                Tag.of(Constants.METRICS_LABEL_ALIAS, meta.getAlias()),
+                                                Tag.of(Constants.METRICS_LABEL_SCHEMA, meta.getSchema()),
+                                                Tag.of(Constants.METRICS_LABEL_TABLE, meta.getTable())
+                                        ),
+                                        new AtomicLong(0)
+                                )
+                        );
+                        AtomicLong timelineFileCountCache = timelineFileCountCacheMap.getIfAbsentPut(
+                                meta.getAlias(),
+                                registry.gauge(
+                                        Constants.METRICS_HUDI_TABLE_TIMELINE_FILE_COUNT,
+                                        Lists.immutable.of(
+                                                Tag.of(Constants.METRICS_LABEL_FLINK_JOB_ID, meta.getJob().getId().toString()),
+                                                Tag.of(Constants.METRICS_LABEL_ALIAS, meta.getAlias()),
+                                                Tag.of(Constants.METRICS_LABEL_SCHEMA, meta.getSchema()),
+                                                Tag.of(Constants.METRICS_LABEL_TABLE, meta.getTable())
+                                        ),
+                                        new AtomicLong(0)
+                                )
+                        );
+
+                        String hdfs = meta.getHudi().getTargetHdfsPath();
+                        if (hudiService.existsHudiTable(hdfs)) {
+                            Long count = hudiService.fileCount(hdfs);
+                            filecountCache.set(count);
+
+                            String timelineHdfs = hdfs + "/.hoodie";
+                            timelineFileCountCache.set(hudiService.fileCount(timelineHdfs));
+                        }
+                    } catch (Exception exception) {
+                        logger.warn("Get file count fail for {}", meta.getAlias(), exception);
+                    }
+                });
+    }
+}

service-monitor/src/main/java/com/lanyuanxiaoyao/service/monitor/metric/PulsarMetrics.java

@@ -75,7 +75,7 @@ public class PulsarMetrics extends Metrics {
                             infoService.savePulsarBacklog(meta.getJob().getId(), meta.getAlias(), backlog);
                         }
                     } catch (Exception exception) {
-                        logger.warn("Update pulsar backlog fail for " + meta.getAlias(), exception);
+                        logger.warn("Update pulsar backlog fail for {}", meta.getAlias(), exception);
                     }
                 });
     }

test/test.http

@@ -134,3 +134,7 @@ GET http://{{username}}:{{password}}@b12s10.hdp.dc:31719/task/law_enforcement?
     pulsar_url=pulsar://132.122.115.158:16650,132.122.115.159:16650,132.122.115.160:16650,132.122.115.161:16650,132.122.115.167:16650,132.122.115.168:16650&
     pulsar_topic=persistent://odcp/acct_sz/acct_item_755&start_time=1716858000000&end_time=1716861600000&
     primary_keys=ACCT_ITEM_ID&partition_keys=ACCT_ID
+
+### Count files
+GET http://{{username}}:{{password}}@b12s20.hdp.dc:19521/hdfs/count?
+    root=hdfs://b2/apps/datalake/hive/dws_wsyyt/external_table_hudi/dws_tb_jt_servuser_data