164 lines
5.6 KiB
Markdown
164 lines
5.6 KiB
Markdown
---
|
||
title: 大数据
|
||
date: 2025-08-03 22:53:51
|
||
tags:
|
||
---
|
||
|
||
# Apache Doris
|
||
|
||
## 简介
|
||
|
||
## 安装
|
||
|
||
### 配置
|
||
|
||
#### Linux系统环境配置（所有节点均需配置）
|
||
|
||
##### 调大文件操作配置
|
||
|
||
```bash
|
||
vim /etc/security/limits.conf
|
||
```
|
||
|
||
在文件最后添加下面几行信息(注意 * 也要复制进去)
|
||
|
||
```conf
|
||
* soft nofile 65536
|
||
* hard nofile 65536
|
||
* soft nproc 65536
|
||
* hard nproc 65536
|
||
```
|
||
|
||
可使用以下命令直接追加到配置文件内
|
||
|
||
```bash
|
||
echo "* soft nofile 65536" >> /etc/security/limits.conf
|
||
echo "* hard nofile 65536" >> /etc/security/limits.conf
|
||
echo "* soft nproc 65536" >> /etc/security/limits.conf
|
||
echo "* hard nproc 65536" >> /etc/security/limits.conf
|
||
```
|
||
|
||
保存完成需`reboot`，也可临时生效，免重启
|
||
|
||
```bash
|
||
ulimit -n 65536
|
||
```
|
||
|
||
# Datax
|
||
|
||
## MySQL离线同步至Apache Doris
|
||
|
||
### 单表同步
|
||
|
||
新建配置文件`mysql_2_doris_t_base_material.json`
|
||
|
||
```json
|
||
{
|
||
"job": {
|
||
"setting": {
|
||
"speed": {
|
||
"channel": 1
|
||
}
|
||
},
|
||
"content": [
|
||
{
|
||
"reader": {
|
||
"name": "mysqlreader",
|
||
"parameter": {
|
||
"username": "root",
|
||
"password": "123456",
|
||
"column": [
|
||
"ID", "PID", "CODE", "SHORTCODE", "NAME", "FULLNAME",
|
||
"ISDETAIL", "CHILDCOUNT", "PYCODE", "LVL", "FLAG", "MODEL",
|
||
"ISSTANDARD", "AUXCLASSID", "ERPCLSID", "UNITGROUPID", "UNITID",
|
||
"DEFAULTLOC", "SPID", "QTYDECIMAL", "SECINV", "MTYPE", "MVER",
|
||
"MATERIAL", "FIRM", "FIRMORDER", "FACEDEAL", "SHAPESIZE",
|
||
"TECHDESC", "UNITWEIGHT", "MCODE", "MCAT", "COSTITEM",
|
||
"ORDERRECTOR", "POHIGHPRICE", "POHGHPRCMNYTYPE", "WWHGHPRC",
|
||
"WWHGHPRCMNYTYPE", "SOLOWPRC", "SOLOWPRCMNYTYPE", "TRACK",
|
||
"PRICEDECIMAL", "ACCTID", "SALEACCTID", "COSTACCTID",
|
||
"DEFAULTROUTINGID", "DEFAULTWORKTYPEID", "PRODUCTPRINCIPAL",
|
||
"PLANNER", "ISBACKFLUSH", "MRPCON", "MRPORDER", "CHARTNUMBER",
|
||
"INSPECTIONLEVEL", "PROCHKMDE", "WWCHKMDE", "SOCHKMDE",
|
||
"WTHDRWCHKMDE", "STKCHKMDE", "OTHERCHKMDE", "INSPECTIONPROJECT",
|
||
"NAMEEN", "MODELEN", "HSNUMBER", "IMPOSTTAXRATE",
|
||
"CONSUMETAXRATE", "STARTSERVICE", "MAKEFILE", "ISFIX",
|
||
"TTERMOFSERVICE", "TTERMOFUSEFULTIME", "PRODUCTNO", "PARAMVALUE",
|
||
"MEMO", "CDATE", "CUSERID", "CSTAFFNAME", "MDATE", "MUSERID",
|
||
"MSTAFFNAME", "K_ID", "K_PID", "DDATE", "DUSERID", "DSTAFFNAME",
|
||
"K_CODE", "AMODEL", "K_COSTITEM", "DWGSYMDESC", "SPDESC",
|
||
"CDRATE", "GWEIGHT", "NWEIGHT", "L", "W", "H", "MSIZE",
|
||
"K_COST", "BATCHAPPLICANT", "MATSLUGGISHSTATUS",
|
||
"SLUGGISHEFFECTIVEDATE", "K_AUTO_ID", "FRATE", "K3CLOUDID"
|
||
],
|
||
"connection": [
|
||
{
|
||
"table": ["t_base_material"],
|
||
"jdbcUrl": ["jdbc:mysql://192.168.6.35:3306/erp?useSSL=false&serverTimezone=Asia/Shanghai"],
|
||
"selectedDatabase": "erp"
|
||
}
|
||
]
|
||
}
|
||
},
|
||
"writer": {
|
||
"name": "doriswriter",
|
||
"parameter": {
|
||
"loadUrl": ["192.168.6.35:8040"],
|
||
"username": "root",
|
||
"password": "",
|
||
"column": [
|
||
"ID", "PID", "CODE", "SHORTCODE", "NAME", "FULLNAME",
|
||
"ISDETAIL", "CHILDCOUNT", "PYCODE", "LVL", "FLAG", "MODEL",
|
||
"ISSTANDARD", "AUXCLASSID", "ERPCLSID", "UNITGROUPID", "UNITID",
|
||
"DEFAULTLOC", "SPID", "QTYDECIMAL", "SECINV", "MTYPE", "MVER",
|
||
"MATERIAL", "FIRM", "FIRMORDER", "FACEDEAL", "SHAPESIZE",
|
||
"TECHDESC", "UNITWEIGHT", "MCODE", "MCAT", "COSTITEM",
|
||
"ORDERRECTOR", "POHIGHPRICE", "POHGHPRCMNYTYPE", "WWHGHPRC",
|
||
"WWHGHPRCMNYTYPE", "SOLOWPRC", "SOLOWPRCMNYTYPE", "TRACK",
|
||
"PRICEDECIMAL", "ACCTID", "SALEACCTID", "COSTACCTID",
|
||
"DEFAULTROUTINGID", "DEFAULTWORKTYPEID", "PRODUCTPRINCIPAL",
|
||
"PLANNER", "ISBACKFLUSH", "MRPCON", "MRPORDER", "CHARTNUMBER",
|
||
"INSPECTIONLEVEL", "PROCHKMDE", "WWCHKMDE", "SOCHKMDE",
|
||
"WTHDRWCHKMDE", "STKCHKMDE", "OTHERCHKMDE", "INSPECTIONPROJECT",
|
||
"NAMEEN", "MODELEN", "HSNUMBER", "IMPOSTTAXRATE",
|
||
"CONSUMETAXRATE", "STARTSERVICE", "MAKEFILE", "ISFIX",
|
||
"TTERMOFSERVICE", "TTERMOFUSEFULTIME", "PRODUCTNO", "PARAMVALUE",
|
||
"MEMO", "CDATE", "CUSERID", "CSTAFFNAME", "MDATE", "MUSERID",
|
||
"MSTAFFNAME", "K_ID", "K_PID", "DDATE", "DUSERID", "DSTAFFNAME",
|
||
"K_CODE", "AMODEL", "K_COSTITEM", "DWGSYMDESC", "SPDESC",
|
||
"CDRATE", "GWEIGHT", "NWEIGHT", "L", "W", "H", "MSIZE",
|
||
"K_COST", "BATCHAPPLICANT", "MATSLUGGISHSTATUS",
|
||
"SLUGGISHEFFECTIVEDATE", "K_AUTO_ID", "FRATE", "K3CLOUDID"
|
||
],
|
||
"postSql": ["select count(1) from t_base_material"],
|
||
"preSql": [],
|
||
"flushInterval":30000,
|
||
"connection": [
|
||
{
|
||
"jdbcUrl": "jdbc:mysql://192.168.6.35:9030/erp",
|
||
"selectedDatabase": "erp",
|
||
"table": ["t_base_material"]
|
||
}
|
||
],
|
||
"loadProps": {
|
||
"format": "json",
|
||
"strip_outer_array":"true",
|
||
"line_delimiter": "\\x02"
|
||
}
|
||
}
|
||
}
|
||
}
|
||
]
|
||
}
|
||
}
|
||
|
||
```
|
||
|
||
执行任务
|
||
|
||
```bash
|
||
cd datax
|
||
python ./bin/datax.py mysql_2_doris_t_base_material.json
|
||
```
|
||
|