feat(all): 增加b9集群相关配置

2024-02-25 12:26:21 +08:00
parent 0338a0d579
commit d0bbdff61d
9 changed files with 77 additions and 18 deletions
--- a/bin/build-all.sh
+++ b/bin/build-all.sh
@@ -19,6 +19,9 @@ ${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service
 mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b2b12
 ${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service-launcher-b2b12-1.0.0-SNAPSHOT.jar

+mvn -pl service-launcher clean package -D skipTests -s ~/.m2/settings-development.xml -P b9b9
+${current_path}/bin/upload-rm.sh ${current_path}/service-launcher/target/service-launcher-b9b9-1.0.0-SNAPSHOT.jar
+
 ${current_path}/bin/upload-rm.sh ${current_path}/service-api/target/service-api-1.0.0-SNAPSHOT.jar
 ${current_path}/bin/upload-rm.sh ${current_path}/service-check/target/service-check-1.0.0-SNAPSHOT.jar
 ${current_path}/bin/upload-rm.sh ${current_path}/service-cli/service-cli-runner/target/service-cli-runner-1.0.0-SNAPSHOT.jar
--- a/service-cli/service-cli-runner/src/main/resources/application-b9.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application-b9.yml
@@ -0,0 +1,32 @@
+deploy:
+  runtime:
+    user: zone_scfp
+    jar-path: /apps/zone_scfp/hudi/jars
+    jdk-path: /opt/jdk8u252-b09/bin/java
+    log-path: /apps/zone_scfp/hudi/logs
+    data-path: /apps/zone_scfp/hudi/data
+    kerberos-keytab-path: /etc/security/keytabs/zone_scfp.app.keytab
+    loki:
+      service-url: http://132.126.207.34:33100
+      service-push-url: ${deploy.runtime.loki.service-url}/loki/api/v1/push
+      hudi-url: http://132.126.207.34:33100
+      hudi-push-url: ${deploy.runtime.loki.hudi-url}/loki/api/v1/push
+    zk-url: b9m1.hdp.dc:2181,b9m2.hdp.dc:2181,b9m3.hdp.dc:2181
+    connector-zk-url: b9m1.hdp.dc:2181,b9m2.hdp.dc:2181,b9m3.hdp.dc:2181
+    hudi:
+      app-hdfs-path: jfs://tdsc/user/zone_scfp/jars/b9
+      archive-hdfs-path: jfs://tdsc/user/zone_scfp/flink/completed-jobs-hudi
+      victoria-push-url: http://132.126.207.34:35710/api/v1/import/prometheus
+      loki-push-url: ${deploy.runtime.loki.hudi-push-url}
+    executor:
+      staging-path: jfs://tdsc/user/zone_scfp/yarn
+      archive-hdfs-path: jfs://tdsc/user/zone_scfp/flink/completed-jobs/
+      task-jar-path: jfs://tdsc/user/zone_scfp/jars/service/service-executor-task-1.0.0-SNAPSHOT.jar
+      task-result-path: jfs://tdsc/user/zone_scfp/task-results
+    security:
+      authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
+      username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
+      darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
+  hosts:
+    - host: b9s1.hdp.dc
+      ip: 132.126.207.34
--- a/service-cli/service-cli-runner/src/main/resources/application.yml
+++ b/service-cli/service-cli-runner/src/main/resources/application.yml
@@ -100,6 +100,26 @@ deploy:
        connector_cluster_sync-queue-name: sync-queue-b12
        connector_cluster_compaction-queue-name: compaction-queue-b12
        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
+    service-launcher-runner-b9:
+      groups:
+        - "service"
+        - "service-hudi"
+        - "service-hudi-launcher"
+      source-jar: service-launcher-runner-b9b9-1.0.0-SNAPSHOT.jar
+      replicas: 6
+      environments:
+        connector_hadoop_kerberos-principal: ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
+        connector_hadoop_kerberos-keytab-path: ${deploy.runtime.kerberos-keytab-path}
+        connector_hudi_app-hdfs-path: ${deploy.runtime.hudi.app-hdfs-path}
+        connector_hudi_archive-hdfs-path: ${deploy.runtime.hudi.archive-hdfs-path}
+        connector_hudi_victoria-push-url: ${deploy.runtime.hudi.victoria-push-url}
+        connector_hudi_loki-push-url: ${deploy.runtime.hudi.loki-push-url}
+      arguments:
+        spring_application_name: service-launcher-runner-b9
+        connector_cluster_cluster: b9
+        connector_cluster_sync-queue-name: sync-queue-b9
+        connector_cluster_compaction-queue-name: compaction-queue-b9
+        connector_zookeeper_connect-url: ${deploy.runtime.connector-zk-url}
    service-info-query:
      order: 1
      groups:
--- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B9LauncherService.java
+++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/launcher/B9LauncherService.java
@@ -0,0 +1,11 @@
+package com.lanyuanxiaoyao.service.forest.service.launcher;
+
+import com.dtflys.forest.annotation.BaseRequest;
+
+/**
+ * @author lanyuanxiaoyao
+ * @date 2023-06-06
+ */
+@BaseRequest(baseURL = "http://service-launcher-runner-b9")
+public interface B9LauncherService extends LauncherService{
+}
--- a/service-web/src/main/resources/static/common/info.js
+++ b/service-web/src/main/resources/static/common/info.js
@@ -3,12 +3,12 @@ function commonInfo() {
        // baseUrl: 'http://132.126.207.131:35690/hudi_services/service_web',
        baseUrl: '/hudi_services/service_web',
        loki: {
-            grafanaUrl: 'http://132.126.207.125:35700',
+            grafanaUrl: 'http://132.126.207.34:35700',
            hudi: {
-                datasource: 'f648174e-7593-45cf-8fe8-8f8d5cf0fdde',
+                datasource: 'e2477661-3c3d-4e3d-a3f7-1d398a5b4b7b',
            },
            service: {
-                datasource: 'b6fee51c-facd-4261-a0eb-8c69a975fba3',
+                datasource: 'fbbc4d0c-16d1-45a3-8ceb-4039b1b9a353',
            }
        },
    }
--- a/service-web/src/main/resources/static/components/overview-tab.js
+++ b/service-web/src/main/resources/static/components/overview-tab.js
@@ -330,7 +330,7 @@ function overviewTab() {
            },
            {type: 'divider'},
            '<span class="italic text-gray-500 my-2">集群 (集群总资源使用,队列资源使用)(调度中任务数,运行中任务数)</span>',
-            overviewYarnJob('b12', 'Sync', undefined, 'default'),
+            overviewYarnJob('b9', 'Sync', undefined, 'default'),
            {type: 'divider'},
            {
                className: 'my-2',
@@ -346,10 +346,7 @@ function overviewTab() {
                ]
            },
            '<span class="italic text-gray-500 my-2">集群 压缩队列任务数(集群总资源使用,队列资源使用)(调度中任务数,运行中任务数)</span>',
-            overviewYarnJob('b12', 'Compaction', 'compaction-queue-b12', 'default'),
-            overviewYarnJob('b1', 'Compaction', 'compaction-queue-b1', 'datalake'),
-            overviewYarnJob('b5', 'Compaction', 'compaction-queue-b5', 'ten_iap.datalake'),
-            overviewYarnJob('a4', 'Compaction', 'compaction-queue-a4', 'ten_iap.datalake'),
+            overviewYarnJob('b9', 'Compaction', 'compaction-queue-b9', 'default'),
            {type: 'divider'},
            {
                type: 'service',
--- a/service-web/src/main/resources/static/components/yarn-cluster-tab.js
+++ b/service-web/src/main/resources/static/components/yarn-cluster-tab.js
@@ -12,14 +12,10 @@ function yarnClusterTab() {
                    {
                        title: '总体情况',
                        tab: [
-                            yarnQueueCrud('b1,b5,a4,b12')
+                            yarnQueueCrud('b9')
                        ],
                    },
-                    yarnTab('b1', 'b1'),
-                    yarnTab('a4', 'a4'),
-                    yarnTab('b5', 'b5'),
-                    yarnTab('t5', 't5'),
-                    yarnTab('b12', 'b12'),
+                    yarnTab('b9', 'b9'),
                ]
            }
        ]
--- a/service-web/src/main/resources/static/extra.html
+++ b/service-web/src/main/resources/static/extra.html
@@ -62,8 +62,8 @@
                    tableTab(),
                    queueTab(),
                    versionTab(),
-                    yarnTab('b12', '同步', undefined, 'Sync'),
-                    yarnTab('b12,b1,b5,a4', '压缩', 'datalake,ten_iap.datalake,default', 'Compaction'),
+                    yarnTab('b9', '同步', undefined, 'Sync'),
+                    yarnTab('b9', '压缩', 'default', 'Compaction'),
                    cloudTab(),
                    yarnClusterTab(),
                    toolTab(),
--- a/service-web/src/main/resources/static/index.html
+++ b/service-web/src/main/resources/static/index.html
@@ -62,8 +62,8 @@
                    tableTab(),
                    queueTab(),
                    versionTab(),
-                    yarnTab('b12', '同步', undefined, 'Sync'),
-                    yarnTab('b12,b1,b5,a4', '压缩', 'datalake,ten_iap.datalake,default', 'Compaction')
+                    yarnTab('b9', '同步', undefined, 'Sync'),
+                    yarnTab('b9', '压缩', 'default', 'Compaction')
                ]
            }
        }