提交
This commit is contained in:
parent
40797fd3c8
commit
3e88c0641a
@ -10,20 +10,22 @@ tags:
|
|||||||
|
|
||||||
1. 首先启动flink 容器 JobManager、TaskManager 两个容器将配置文件复制出来方便挂载
|
1. 首先启动flink 容器 JobManager、TaskManager 两个容器将配置文件复制出来方便挂载
|
||||||
```shell
|
```shell
|
||||||
|
docker network create flink-network
|
||||||
|
|
||||||
docker run \
|
docker run \
|
||||||
-itd \
|
-itd \
|
||||||
--name=jobmanager \
|
--name=jobmanager \
|
||||||
--publish 8081:8081 \
|
--publish 8081:8081 \
|
||||||
--network flink-network \
|
--network flink-network \
|
||||||
--env FLINK_PROPERTIES="jobmanager.rpc.address: jobmanager" \
|
--env FLINK_PROPERTIES="jobmanager.rpc.address: jobmanager" \
|
||||||
flink:scala_2.12-java8 jobmanager
|
flink:1.16.0-scala_2.12-java8 jobmanager
|
||||||
|
|
||||||
docker run \
|
docker run \
|
||||||
-itd \
|
-itd \
|
||||||
--name=taskmanager \
|
--name=taskmanager \
|
||||||
--network flink-network \
|
--network flink-network \
|
||||||
--env FLINK_PROPERTIES="jobmanager.rpc.address: jobmanager" \
|
--env FLINK_PROPERTIES="jobmanager.rpc.address: jobmanager" \
|
||||||
flink:scala_2.12-java8 taskmanager
|
flink:1.16.0-scala_2.12-java8 taskmanager
|
||||||
|
|
||||||
|
|
||||||
```
|
```
|
||||||
@ -44,11 +46,13 @@ tags:
|
|||||||
```
|
```
|
||||||
|
|
||||||
3. 搭建集群
|
3. 搭建集群
|
||||||
|
docker-compose.yaml 文件
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
version: "2.2"
|
version: "2.2"
|
||||||
services:
|
services:
|
||||||
jobmanager:
|
jobmanager:
|
||||||
image: flink:scala_2.12-java8
|
image: flink:1.16.0-scala_2.12-java8
|
||||||
container_name: jobmanager-1
|
container_name: jobmanager-1
|
||||||
expose:
|
expose:
|
||||||
- "6123"
|
- "6123"
|
||||||
@ -68,7 +72,7 @@ tags:
|
|||||||
networks:
|
networks:
|
||||||
- flink-network
|
- flink-network
|
||||||
taskmanager:
|
taskmanager:
|
||||||
image: flink:scala_2.12-java8
|
image: flink:1.16.0-scala_2.12-java8
|
||||||
container_name: taskmanager-1
|
container_name: taskmanager-1
|
||||||
depends_on:
|
depends_on:
|
||||||
- jobmanager
|
- jobmanager
|
||||||
@ -82,7 +86,7 @@ tags:
|
|||||||
- |
|
- |
|
||||||
FLINK_PROPERTIES=
|
FLINK_PROPERTIES=
|
||||||
jobmanager.rpc.address: jobmanager
|
jobmanager.rpc.address: jobmanager
|
||||||
taskmanager.numberOfTaskSlots: 2
|
taskmanager.numberOfTaskSlots: 8
|
||||||
parallelism.default: 2
|
parallelism.default: 2
|
||||||
networks:
|
networks:
|
||||||
- flink-network
|
- flink-network
|
||||||
@ -91,5 +95,91 @@ tags:
|
|||||||
external: true
|
external: true
|
||||||
```
|
```
|
||||||
|
|
||||||
|
4. 启动集群
|
||||||
|
```shell
|
||||||
|
docker-compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Kafka 集群
|
||||||
|
|
||||||
|
```shell
|
||||||
|
docker network create kfk-network
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
docker -compose.yaml
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
kafka1:
|
||||||
|
image: 'bitnami/kafka:3.6.1'
|
||||||
|
ports:
|
||||||
|
- '19092:19092'
|
||||||
|
environment:
|
||||||
|
- KAFKA_KRAFT_CLUSTER_ID=EX5bq5NfRe2IX1nhxrSO6g
|
||||||
|
- KAFKA_CFG_NODE_ID=1
|
||||||
|
- KAFKA_CFG_PROCESS_ROLES=broker,controller
|
||||||
|
- KAFKA_CFG_LISTENERS=INTERNAL://:9092, EXTERNAL://:19092, CONTROLLER://:9093
|
||||||
|
- KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka1:9092, EXTERNAL://<宿主机IP>:19092
|
||||||
|
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT, EXTERNAL:PLAINTEXT, CONTROLLER:PLAINTEXT
|
||||||
|
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093, 2@kafka2:9093, 3@kafka3:9093
|
||||||
|
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
|
||||||
|
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
|
||||||
|
networks:
|
||||||
|
- kfk-network
|
||||||
|
kafka2:
|
||||||
|
image: 'bitnami/kafka:3.6.1'
|
||||||
|
ports:
|
||||||
|
- '29092:19092'
|
||||||
|
environment:
|
||||||
|
- KAFKA_KRAFT_CLUSTER_ID=EX5bq5NfRe2IX1nhxrSO6g
|
||||||
|
- KAFKA_CFG_NODE_ID=2
|
||||||
|
- KAFKA_CFG_PROCESS_ROLES=broker,controller
|
||||||
|
- KAFKA_CFG_LISTENERS=INTERNAL://:9092, EXTERNAL://:19092, CONTROLLER://:9093
|
||||||
|
- KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka2:9092, EXTERNAL://<宿主机IP>:29092
|
||||||
|
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT, EXTERNAL:PLAINTEXT, CONTROLLER:PLAINTEXT
|
||||||
|
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093, 2@kafka2:9093, 3@kafka3:9093
|
||||||
|
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
|
||||||
|
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
|
||||||
|
networks:
|
||||||
|
- kfk-network
|
||||||
|
kafka3:
|
||||||
|
image: 'bitnami/kafka:3.6.1'
|
||||||
|
ports:
|
||||||
|
- '39092:19092'
|
||||||
|
environment:
|
||||||
|
- KAFKA_KRAFT_CLUSTER_ID=EX5bq5NfRe2IX1nhxrSO6g
|
||||||
|
- KAFKA_CFG_NODE_ID=3
|
||||||
|
- KAFKA_CFG_PROCESS_ROLES=broker,controller
|
||||||
|
- KAFKA_CFG_LISTENERS=INTERNAL://:9092, EXTERNAL://:19092, CONTROLLER://:9093
|
||||||
|
- KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka3:9092, EXTERNAL://<宿主机IP>:39092
|
||||||
|
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT, EXTERNAL:PLAINTEXT, CONTROLLER:PLAINTEXT
|
||||||
|
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093, 2@kafka2:9093, 3@kafka3:9093
|
||||||
|
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
|
||||||
|
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
|
||||||
|
networks:
|
||||||
|
- kfk-network
|
||||||
|
kafka-ui:
|
||||||
|
image: provectuslabs/kafka-ui:master
|
||||||
|
container_name: kafka-ui
|
||||||
|
ports:
|
||||||
|
- "38080:8080"
|
||||||
|
restart: always
|
||||||
|
environment:
|
||||||
|
- KAFKA_CLUSTERS_0_NAME=local
|
||||||
|
- KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka1:9092,kafka2:9092,kafka3:9092
|
||||||
|
- KAFKA_CLUSTERS_0_READONLY=true
|
||||||
|
depends_on:
|
||||||
|
- kafka1
|
||||||
|
- kafka2
|
||||||
|
- kafka3
|
||||||
|
networks:
|
||||||
|
- kfk-network
|
||||||
|
networks:
|
||||||
|
kfk-network:
|
||||||
|
external: true
|
||||||
|
```
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user