diff --git a/source/_posts/Docker-Compose.md b/source/_posts/Docker-Compose.md index 80ac6f4..7d75108 100644 --- a/source/_posts/Docker-Compose.md +++ b/source/_posts/Docker-Compose.md @@ -10,20 +10,22 @@ tags: 1. 首先启动flink 容器 JobManager、TaskManager 两个容器将配置文件复制出来方便挂载 ```shell + docker network create flink-network + docker run \ -itd \ --name=jobmanager \ --publish 8081:8081 \ --network flink-network \ --env FLINK_PROPERTIES="jobmanager.rpc.address: jobmanager" \ - flink:scala_2.12-java8 jobmanager + flink:1.16.0-scala_2.12-java8 jobmanager docker run \ -itd \ --name=taskmanager \ --network flink-network \ --env FLINK_PROPERTIES="jobmanager.rpc.address: jobmanager" \ - flink:scala_2.12-java8 taskmanager + flink:1.16.0-scala_2.12-java8 taskmanager ``` @@ -44,11 +46,13 @@ tags: ``` 3. 搭建集群 + docker-compose.yaml 文件 + ```yaml version: "2.2" services: jobmanager: - image: flink:scala_2.12-java8 + image: flink:1.16.0-scala_2.12-java8 container_name: jobmanager-1 expose: - "6123" @@ -68,7 +72,7 @@ tags: networks: - flink-network taskmanager: - image: flink:scala_2.12-java8 + image: flink:1.16.0-scala_2.12-java8 container_name: taskmanager-1 depends_on: - jobmanager @@ -82,7 +86,7 @@ tags: - | FLINK_PROPERTIES= jobmanager.rpc.address: jobmanager - taskmanager.numberOfTaskSlots: 2 + taskmanager.numberOfTaskSlots: 8 parallelism.default: 2 networks: - flink-network @@ -91,5 +95,91 @@ tags: external: true ``` +4. 启动集群 + ```shell + docker-compose up -d + ``` + +## Kafka 集群 + +```shell +docker network create kfk-network +``` + + + +docker -compose.yaml + +```yaml +services: + kafka1: + image: 'bitnami/kafka:3.6.1' + ports: + - '19092:19092' + environment: + - KAFKA_KRAFT_CLUSTER_ID=EX5bq5NfRe2IX1nhxrSO6g + - KAFKA_CFG_NODE_ID=1 + - KAFKA_CFG_PROCESS_ROLES=broker,controller + - KAFKA_CFG_LISTENERS=INTERNAL://:9092, EXTERNAL://:19092, CONTROLLER://:9093 + - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka1:9092, EXTERNAL://<宿主机IP>:19092 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT, EXTERNAL:PLAINTEXT, CONTROLLER:PLAINTEXT + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093, 2@kafka2:9093, 3@kafka3:9093 + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL + networks: + - kfk-network + kafka2: + image: 'bitnami/kafka:3.6.1' + ports: + - '29092:19092' + environment: + - KAFKA_KRAFT_CLUSTER_ID=EX5bq5NfRe2IX1nhxrSO6g + - KAFKA_CFG_NODE_ID=2 + - KAFKA_CFG_PROCESS_ROLES=broker,controller + - KAFKA_CFG_LISTENERS=INTERNAL://:9092, EXTERNAL://:19092, CONTROLLER://:9093 + - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka2:9092, EXTERNAL://<宿主机IP>:29092 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT, EXTERNAL:PLAINTEXT, CONTROLLER:PLAINTEXT + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093, 2@kafka2:9093, 3@kafka3:9093 + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL + networks: + - kfk-network + kafka3: + image: 'bitnami/kafka:3.6.1' + ports: + - '39092:19092' + environment: + - KAFKA_KRAFT_CLUSTER_ID=EX5bq5NfRe2IX1nhxrSO6g + - KAFKA_CFG_NODE_ID=3 + - KAFKA_CFG_PROCESS_ROLES=broker,controller + - KAFKA_CFG_LISTENERS=INTERNAL://:9092, EXTERNAL://:19092, CONTROLLER://:9093 + - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka3:9092, EXTERNAL://<宿主机IP>:39092 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT, EXTERNAL:PLAINTEXT, CONTROLLER:PLAINTEXT + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093, 2@kafka2:9093, 3@kafka3:9093 + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL + networks: + - kfk-network + kafka-ui: + image: provectuslabs/kafka-ui:master + container_name: kafka-ui + ports: + - "38080:8080" + restart: always + environment: + - KAFKA_CLUSTERS_0_NAME=local + - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka1:9092,kafka2:9092,kafka3:9092 + - KAFKA_CLUSTERS_0_READONLY=true + depends_on: + - kafka1 + - kafka2 + - kafka3 + networks: + - kfk-network +networks: + kfk-network: + external: true +``` +