XiaodaBlogSource/source/_posts/大数据.md

title, date, tags

title	date	tags
大数据	2025-08-03 22:53:51

Apache Doris

简介

安装

配置

Linux系统环境配置（所有节点均需配置）

调大文件操作配置

vim /etc/security/limits.conf

在文件最后添加下面几行信息(注意 * 也要复制进去)

* soft nofile 65536
* hard nofile 65536
* soft nproc 65536
* hard nproc 65536

可使用以下命令直接追加到配置文件内

echo "* soft nofile 65536" >> /etc/security/limits.conf
echo "* hard nofile 65536" >> /etc/security/limits.conf
echo "* soft nproc 65536" >> /etc/security/limits.conf
echo "* hard nproc 65536" >> /etc/security/limits.conf

保存完成需reboot，也可临时生效，免重启

ulimit -n 65536

Datax

MySQL离线同步至Apache Doris

单表同步

新建配置文件mysql_2_doris_t_base_material.json

{
  "job": {
    "setting": {
      "speed": {
        "channel": 1
      }
    },
    "content": [
      {
        "reader": {
          "name": "mysqlreader",
          "parameter": {
            "username": "root",
            "password": "123456",
            "column": [
              "ID", "PID", "CODE", "SHORTCODE", "NAME", "FULLNAME",
              "ISDETAIL", "CHILDCOUNT", "PYCODE", "LVL", "FLAG", "MODEL",
              "ISSTANDARD", "AUXCLASSID", "ERPCLSID", "UNITGROUPID", "UNITID",
              "DEFAULTLOC", "SPID", "QTYDECIMAL", "SECINV", "MTYPE", "MVER",
              "MATERIAL", "FIRM", "FIRMORDER", "FACEDEAL", "SHAPESIZE",
              "TECHDESC", "UNITWEIGHT", "MCODE", "MCAT", "COSTITEM",
              "ORDERRECTOR", "POHIGHPRICE", "POHGHPRCMNYTYPE", "WWHGHPRC",
              "WWHGHPRCMNYTYPE", "SOLOWPRC", "SOLOWPRCMNYTYPE", "TRACK",
              "PRICEDECIMAL", "ACCTID", "SALEACCTID", "COSTACCTID",
              "DEFAULTROUTINGID", "DEFAULTWORKTYPEID", "PRODUCTPRINCIPAL",
              "PLANNER", "ISBACKFLUSH", "MRPCON", "MRPORDER", "CHARTNUMBER",
              "INSPECTIONLEVEL", "PROCHKMDE", "WWCHKMDE", "SOCHKMDE",
              "WTHDRWCHKMDE", "STKCHKMDE", "OTHERCHKMDE", "INSPECTIONPROJECT",
              "NAMEEN", "MODELEN", "HSNUMBER", "IMPOSTTAXRATE",
              "CONSUMETAXRATE", "STARTSERVICE", "MAKEFILE", "ISFIX",
              "TTERMOFSERVICE", "TTERMOFUSEFULTIME", "PRODUCTNO", "PARAMVALUE",
              "MEMO", "CDATE", "CUSERID", "CSTAFFNAME", "MDATE", "MUSERID",
              "MSTAFFNAME", "K_ID", "K_PID", "DDATE", "DUSERID", "DSTAFFNAME",
              "K_CODE", "AMODEL", "K_COSTITEM", "DWGSYMDESC", "SPDESC",
              "CDRATE", "GWEIGHT", "NWEIGHT", "L", "W", "H", "MSIZE",
              "K_COST", "BATCHAPPLICANT", "MATSLUGGISHSTATUS",
              "SLUGGISHEFFECTIVEDATE", "K_AUTO_ID", "FRATE", "K3CLOUDID"
            ],
            "connection": [
              {
                "table": ["t_base_material"],
                "jdbcUrl": ["jdbc:mysql://192.168.6.35:3306/erp?useSSL=false&serverTimezone=Asia/Shanghai"],
                "selectedDatabase": "erp"
              }
            ]
          }
        },
        "writer": {
          "name": "doriswriter",
          "parameter": {
            "loadUrl": ["192.168.6.35:8040"],
            "username": "root",
            "password": "",
            "column": [
              "ID", "PID", "CODE", "SHORTCODE", "NAME", "FULLNAME",
              "ISDETAIL", "CHILDCOUNT", "PYCODE", "LVL", "FLAG", "MODEL",
              "ISSTANDARD", "AUXCLASSID", "ERPCLSID", "UNITGROUPID", "UNITID",
              "DEFAULTLOC", "SPID", "QTYDECIMAL", "SECINV", "MTYPE", "MVER",
              "MATERIAL", "FIRM", "FIRMORDER", "FACEDEAL", "SHAPESIZE",
              "TECHDESC", "UNITWEIGHT", "MCODE", "MCAT", "COSTITEM",
              "ORDERRECTOR", "POHIGHPRICE", "POHGHPRCMNYTYPE", "WWHGHPRC",
              "WWHGHPRCMNYTYPE", "SOLOWPRC", "SOLOWPRCMNYTYPE", "TRACK",
              "PRICEDECIMAL", "ACCTID", "SALEACCTID", "COSTACCTID",
              "DEFAULTROUTINGID", "DEFAULTWORKTYPEID", "PRODUCTPRINCIPAL",
              "PLANNER", "ISBACKFLUSH", "MRPCON", "MRPORDER", "CHARTNUMBER",
              "INSPECTIONLEVEL", "PROCHKMDE", "WWCHKMDE", "SOCHKMDE",
              "WTHDRWCHKMDE", "STKCHKMDE", "OTHERCHKMDE", "INSPECTIONPROJECT",
              "NAMEEN", "MODELEN", "HSNUMBER", "IMPOSTTAXRATE",
              "CONSUMETAXRATE", "STARTSERVICE", "MAKEFILE", "ISFIX",
              "TTERMOFSERVICE", "TTERMOFUSEFULTIME", "PRODUCTNO", "PARAMVALUE",
              "MEMO", "CDATE", "CUSERID", "CSTAFFNAME", "MDATE", "MUSERID",
              "MSTAFFNAME", "K_ID", "K_PID", "DDATE", "DUSERID", "DSTAFFNAME",
              "K_CODE", "AMODEL", "K_COSTITEM", "DWGSYMDESC", "SPDESC",
              "CDRATE", "GWEIGHT", "NWEIGHT", "L", "W", "H", "MSIZE",
              "K_COST", "BATCHAPPLICANT", "MATSLUGGISHSTATUS",
              "SLUGGISHEFFECTIVEDATE", "K_AUTO_ID", "FRATE", "K3CLOUDID"
            ],
            "postSql": ["select count(1) from t_base_material"],
            "preSql": [],
            "flushInterval":30000,
            "connection": [
                    {
                        "jdbcUrl": "jdbc:mysql://192.168.6.35:9030/erp",
                        "selectedDatabase": "erp",
                        "table": ["t_base_material"]
                    }
            ],
            "loadProps": {
                        "format": "json",
                        "strip_outer_array":"true",
                        "line_delimiter": "\\x02"
                       }
          }
        }
      }
    ]
  }
}
                                                                            

执行任务

cd datax
python ./bin/datax.py mysql_2_doris_t_base_material.json