init: 初始化 XiaowenBlog 博客项目，添加配置文件及技术文章
This commit is contained in:
@@ -0,0 +1,353 @@
|
||||
---
|
||||
title: 大数据
|
||||
date: 2025-08-03 22:53:51
|
||||
tags:
|
||||
---
|
||||
|
||||
# Apache Doris
|
||||
|
||||
## 简介
|
||||
|
||||
## 安装
|
||||
|
||||
### Docker Compose 方式
|
||||
|
||||
> [Quick Start - Apache Doris](https://doris.apache.org/docs/dev/gettingStarted/quick-start)
|
||||
|
||||
新建用于Docker Compose 集群编排启动目录`mkdir -p /OLAP/doris`
|
||||
|
||||
```bash
|
||||
mkdir -p /OLAP/doris
|
||||
```
|
||||
|
||||
编写启动脚本`start-doris.sh`
|
||||
|
||||
> https://doris.apache.org/files/start-doris.sh
|
||||
|
||||
```sh
|
||||
#!/bin/bash
|
||||
|
||||
# Default version
|
||||
DORIS_QUICK_START_VERSION="2.1.9"
|
||||
|
||||
# Parse parameters
|
||||
while getopts "v:" opt; do
|
||||
case $opt in
|
||||
v) DORIS_QUICK_START_VERSION="$OPTARG"
|
||||
;;
|
||||
\?) echo "Invalid option: -$OPTARG" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Check system type
|
||||
OS_TYPE=$(uname -s)
|
||||
if [[ "$OS_TYPE" != "Linux" && "$OS_TYPE" != "Darwin" ]]; then
|
||||
echo "Error: Unsupported operating system [$OS_TYPE], only Linux and Mac are supported"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check Docker environment
|
||||
if ! command -v docker &> /dev/null; then
|
||||
echo "Error: Docker environment not detected, please install Docker first"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check docker-compose
|
||||
COMPOSE_CMD=""
|
||||
if command -v docker-compose &> /dev/null; then
|
||||
COMPOSE_CMD="docker-compose"
|
||||
elif docker compose version &> /dev/null; then
|
||||
COMPOSE_CMD="docker compose"
|
||||
else
|
||||
echo "Error: docker-compose plugin or docker-compose command is required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Generate docker-compose configuration for corresponding system
|
||||
if [[ "$OS_TYPE" == "Linux" ]]; then
|
||||
cat > docker-compose-doris.yaml <<EOF
|
||||
version: "3"
|
||||
services:
|
||||
fe:
|
||||
image: apache/doris:fe-${DORIS_QUICK_START_VERSION}
|
||||
hostname: fe
|
||||
environment:
|
||||
- FE_SERVERS=fe1:127.0.0.1:9010
|
||||
- FE_ID=1
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /mnt/app/doris/fe/doris-meta/:/opt/apache-doris/fe/doris-meta/
|
||||
- /mnt/app/doris/fe/log/:/opt/apache-doris/fe/log/
|
||||
be:
|
||||
image: apache/doris:be-${DORIS_QUICK_START_VERSION}
|
||||
hostname: be
|
||||
environment:
|
||||
- FE_SERVERS=fe1:127.0.0.1:9010
|
||||
- BE_ADDR=127.0.0.1:9050
|
||||
depends_on:
|
||||
- fe
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /mnt/app/doris/be/storage/:/opt/apache-doris/be/storage/
|
||||
- /mnt/app/doris/be/script/:/docker-entrypoint-initdb.d/
|
||||
EOF
|
||||
else # Mac system
|
||||
cat > docker-compose-doris.yaml <<EOF
|
||||
version: "3"
|
||||
networks:
|
||||
custom_network:
|
||||
driver: bridge
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 172.20.80.0/24
|
||||
|
||||
services:
|
||||
fe:
|
||||
image: apache/doris:fe-${DORIS_QUICK_START_VERSION}
|
||||
hostname: fe
|
||||
ports:
|
||||
- 8030:8030
|
||||
- 9030:9030
|
||||
- 9010:9010
|
||||
environment:
|
||||
- FE_SERVERS=fe1:172.20.80.2:9010
|
||||
- FE_ID=1
|
||||
networks:
|
||||
custom_network:
|
||||
ipv4_address: 172.20.80.2
|
||||
|
||||
be:
|
||||
image: apache/doris:be-${DORIS_QUICK_START_VERSION}
|
||||
hostname: be
|
||||
ports:
|
||||
- 8040:8040
|
||||
- 9050:9050
|
||||
environment:
|
||||
- FE_SERVERS=fe1:172.20.80.2:9010
|
||||
- BE_ADDR=172.20.80.3:9050
|
||||
depends_on:
|
||||
- fe
|
||||
networks:
|
||||
custom_network:
|
||||
ipv4_address: 172.20.80.3
|
||||
EOF
|
||||
fi
|
||||
|
||||
# Start services
|
||||
$COMPOSE_CMD -f docker-compose-doris.yaml up -d
|
||||
|
||||
echo "Doris cluster started successfully, version: ${DORIS_QUICK_START_VERSION}"
|
||||
echo "You can manage the cluster using the following commands:"
|
||||
echo " Stop cluster: $COMPOSE_CMD -f docker-compose-doris.yaml down"
|
||||
echo " View logs: $COMPOSE_CMD -f docker-compose-doris.yaml logs -f"
|
||||
echo " Connect to cluster: mysql -uroot -P9030 -h127.0.0.1"
|
||||
|
||||
# Display connection information based on system type
|
||||
if [[ "$OS_TYPE" == "Linux" ]]; then
|
||||
echo -e "\nAccess FE/BE http ports (8030, 8040) using the following addresses (Linux system):"
|
||||
echo " http://127.0.0.1:8030"
|
||||
echo " http://127.0.0.1:8040"
|
||||
elif [[ "$OS_TYPE" == "Darwin" ]]; then
|
||||
echo -e "\nAccess FE/BE http ports (8030, 8040) using the following addresses (Mac system):"
|
||||
echo " http://docker.for.mac.localhost:8030"
|
||||
echo " http://docker.for.mac.localhost:8040"
|
||||
echo "Note: If access fails, try using 127.0.0.1 address:"
|
||||
echo " http://127.0.0.1:8030"
|
||||
echo " http://127.0.0.1:8040"
|
||||
fi
|
||||
```
|
||||
|
||||
赋予权限
|
||||
|
||||
```bash
|
||||
chmod 755 start-doris.sh
|
||||
```
|
||||
|
||||
启动集群
|
||||
|
||||
```bash
|
||||
./start-doris.sh
|
||||
```
|
||||
|
||||
可以指定启动的版本通过`-v`参数，比如：
|
||||
|
||||
```bash
|
||||
./start-doris.sh -v 2.1.8
|
||||
```
|
||||
|
||||
使用MySQL客户端连接到集群并检查集群状态
|
||||
|
||||
```bash
|
||||
## Check the FE status to ensure that both the Join and Alive columns are true.
|
||||
mysql -uroot -P9030 -h127.0.0.1 -e 'SELECT `host`, `join`, `alive` FROM frontends()'
|
||||
+-----------+------+-------+
|
||||
| host | join | alive |
|
||||
+-----------+------+-------+
|
||||
| 127.0.0.1 | true | true |
|
||||
+-----------+------+-------+
|
||||
|
||||
## Check the BE status to ensure that the Alive column is true.
|
||||
mysql -uroot -P9030 -h127.0.0.1 -e 'SELECT `host`, `alive` FROM backends()'
|
||||
+-----------+-------+
|
||||
| host | alive |
|
||||
+-----------+-------+
|
||||
| 127.0.0.1 | 1 |
|
||||
+-----------+-------+
|
||||
|
||||
```
|
||||
|
||||
|
||||
|
||||
### 配置
|
||||
|
||||
#### Linux系统环境配置（所有节点均需配置）
|
||||
|
||||
##### 调大文件操作配置
|
||||
|
||||
```bash
|
||||
vim /etc/security/limits.conf
|
||||
```
|
||||
|
||||
在文件最后添加下面几行信息(注意 * 也要复制进去)
|
||||
|
||||
```conf
|
||||
* soft nofile 65536
|
||||
* hard nofile 65536
|
||||
* soft nproc 65536
|
||||
* hard nproc 65536
|
||||
```
|
||||
|
||||
可使用以下命令直接追加到配置文件内
|
||||
|
||||
```bash
|
||||
echo "* soft nofile 65536" >> /etc/security/limits.conf
|
||||
echo "* hard nofile 65536" >> /etc/security/limits.conf
|
||||
echo "* soft nproc 65536" >> /etc/security/limits.conf
|
||||
echo "* hard nproc 65536" >> /etc/security/limits.conf
|
||||
```
|
||||
|
||||
保存完成需`reboot`，也可临时生效，免重启
|
||||
|
||||
```bash
|
||||
ulimit -n 65536
|
||||
```
|
||||
|
||||
# Datax
|
||||
|
||||
## MySQL离线同步至Apache Doris
|
||||
|
||||
### 单表同步
|
||||
|
||||
新建配置文件`mysql_2_doris_t_base_material.json`
|
||||
|
||||
```json
|
||||
{
|
||||
"job": {
|
||||
"setting": {
|
||||
"speed": {
|
||||
"channel": 1
|
||||
}
|
||||
},
|
||||
"content": [
|
||||
{
|
||||
"reader": {
|
||||
"name": "mysqlreader",
|
||||
"parameter": {
|
||||
"username": "root",
|
||||
"password": "123456",
|
||||
"column": [
|
||||
"ID", "PID", "CODE", "SHORTCODE", "NAME", "FULLNAME",
|
||||
"ISDETAIL", "CHILDCOUNT", "PYCODE", "LVL", "FLAG", "MODEL",
|
||||
"ISSTANDARD", "AUXCLASSID", "ERPCLSID", "UNITGROUPID", "UNITID",
|
||||
"DEFAULTLOC", "SPID", "QTYDECIMAL", "SECINV", "MTYPE", "MVER",
|
||||
"MATERIAL", "FIRM", "FIRMORDER", "FACEDEAL", "SHAPESIZE",
|
||||
"TECHDESC", "UNITWEIGHT", "MCODE", "MCAT", "COSTITEM",
|
||||
"ORDERRECTOR", "POHIGHPRICE", "POHGHPRCMNYTYPE", "WWHGHPRC",
|
||||
"WWHGHPRCMNYTYPE", "SOLOWPRC", "SOLOWPRCMNYTYPE", "TRACK",
|
||||
"PRICEDECIMAL", "ACCTID", "SALEACCTID", "COSTACCTID",
|
||||
"DEFAULTROUTINGID", "DEFAULTWORKTYPEID", "PRODUCTPRINCIPAL",
|
||||
"PLANNER", "ISBACKFLUSH", "MRPCON", "MRPORDER", "CHARTNUMBER",
|
||||
"INSPECTIONLEVEL", "PROCHKMDE", "WWCHKMDE", "SOCHKMDE",
|
||||
"WTHDRWCHKMDE", "STKCHKMDE", "OTHERCHKMDE", "INSPECTIONPROJECT",
|
||||
"NAMEEN", "MODELEN", "HSNUMBER", "IMPOSTTAXRATE",
|
||||
"CONSUMETAXRATE", "STARTSERVICE", "MAKEFILE", "ISFIX",
|
||||
"TTERMOFSERVICE", "TTERMOFUSEFULTIME", "PRODUCTNO", "PARAMVALUE",
|
||||
"MEMO", "CDATE", "CUSERID", "CSTAFFNAME", "MDATE", "MUSERID",
|
||||
"MSTAFFNAME", "K_ID", "K_PID", "DDATE", "DUSERID", "DSTAFFNAME",
|
||||
"K_CODE", "AMODEL", "K_COSTITEM", "DWGSYMDESC", "SPDESC",
|
||||
"CDRATE", "GWEIGHT", "NWEIGHT", "L", "W", "H", "MSIZE",
|
||||
"K_COST", "BATCHAPPLICANT", "MATSLUGGISHSTATUS",
|
||||
"SLUGGISHEFFECTIVEDATE", "K_AUTO_ID", "FRATE", "K3CLOUDID"
|
||||
],
|
||||
"connection": [
|
||||
{
|
||||
"table": ["t_base_material"],
|
||||
"jdbcUrl": ["jdbc:mysql://192.168.6.35:3306/erp?useSSL=false&serverTimezone=Asia/Shanghai"],
|
||||
"selectedDatabase": "erp"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"writer": {
|
||||
"name": "doriswriter",
|
||||
"parameter": {
|
||||
"loadUrl": ["192.168.6.35:8040"],
|
||||
"username": "root",
|
||||
"password": "",
|
||||
"column": [
|
||||
"ID", "PID", "CODE", "SHORTCODE", "NAME", "FULLNAME",
|
||||
"ISDETAIL", "CHILDCOUNT", "PYCODE", "LVL", "FLAG", "MODEL",
|
||||
"ISSTANDARD", "AUXCLASSID", "ERPCLSID", "UNITGROUPID", "UNITID",
|
||||
"DEFAULTLOC", "SPID", "QTYDECIMAL", "SECINV", "MTYPE", "MVER",
|
||||
"MATERIAL", "FIRM", "FIRMORDER", "FACEDEAL", "SHAPESIZE",
|
||||
"TECHDESC", "UNITWEIGHT", "MCODE", "MCAT", "COSTITEM",
|
||||
"ORDERRECTOR", "POHIGHPRICE", "POHGHPRCMNYTYPE", "WWHGHPRC",
|
||||
"WWHGHPRCMNYTYPE", "SOLOWPRC", "SOLOWPRCMNYTYPE", "TRACK",
|
||||
"PRICEDECIMAL", "ACCTID", "SALEACCTID", "COSTACCTID",
|
||||
"DEFAULTROUTINGID", "DEFAULTWORKTYPEID", "PRODUCTPRINCIPAL",
|
||||
"PLANNER", "ISBACKFLUSH", "MRPCON", "MRPORDER", "CHARTNUMBER",
|
||||
"INSPECTIONLEVEL", "PROCHKMDE", "WWCHKMDE", "SOCHKMDE",
|
||||
"WTHDRWCHKMDE", "STKCHKMDE", "OTHERCHKMDE", "INSPECTIONPROJECT",
|
||||
"NAMEEN", "MODELEN", "HSNUMBER", "IMPOSTTAXRATE",
|
||||
"CONSUMETAXRATE", "STARTSERVICE", "MAKEFILE", "ISFIX",
|
||||
"TTERMOFSERVICE", "TTERMOFUSEFULTIME", "PRODUCTNO", "PARAMVALUE",
|
||||
"MEMO", "CDATE", "CUSERID", "CSTAFFNAME", "MDATE", "MUSERID",
|
||||
"MSTAFFNAME", "K_ID", "K_PID", "DDATE", "DUSERID", "DSTAFFNAME",
|
||||
"K_CODE", "AMODEL", "K_COSTITEM", "DWGSYMDESC", "SPDESC",
|
||||
"CDRATE", "GWEIGHT", "NWEIGHT", "L", "W", "H", "MSIZE",
|
||||
"K_COST", "BATCHAPPLICANT", "MATSLUGGISHSTATUS",
|
||||
"SLUGGISHEFFECTIVEDATE", "K_AUTO_ID", "FRATE", "K3CLOUDID"
|
||||
],
|
||||
"postSql": ["select count(1) from t_base_material"],
|
||||
"preSql": [],
|
||||
"flushInterval":30000,
|
||||
"connection": [
|
||||
{
|
||||
"jdbcUrl": "jdbc:mysql://192.168.6.35:9030/erp",
|
||||
"selectedDatabase": "erp",
|
||||
"table": ["t_base_material"]
|
||||
}
|
||||
],
|
||||
"loadProps": {
|
||||
"format": "json",
|
||||
"strip_outer_array":"true",
|
||||
"line_delimiter": "\\x02"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
执行任务
|
||||
|
||||
```bash
|
||||
cd datax
|
||||
python ./bin/datax.py mysql_2_doris_t_base_material.json
|
||||
```
|
||||
|
||||
Reference in New Issue
Block a user