Ali Open Source Data Synchronization Tool--DataX

 

Ali Open Source Data Synchronization Tool--DataX

What is it? :

Offline Synchronization Tool for Heterogeneous Data Sources

What can we do? :

It can synchronize data between MySQL sqlServer Oracle Hive HBase FTP stably and efficiently.

 

Design ideas:

Network Connection - "Star Connection"

 

Look at the picture:

 

 

What data synchronization is currently supported? :

 

Core architecture:

Recommend Python 2.67 instead of python 3,0

Use methods and cases:

 

1. Prepare a job.json or job.xml

Execute orders:

 

python datax.py job.json

 

 

Download address: QuickStart


Provide several json s:

1: oracle to local file

 

{  
    "job": {  
        "content": [  
            {  
                 "reader": {
                    "name": "oraclereader",
                    "parameter": {
                        "username": "xxxxx",
                        "password": "xxxxxx",
                        "where": "",
                        "connection": [
                            {
                                "querySql": [
                                    "select * from table1"
                                ],
                                "jdbcUrl": [
                                    "jdbc:oracle:thin:@192.168.121.30:1521:tcenter"
                                ]
                            }
                        ]
                    }
                },  
                "writer": {  
                    "name": "txtfilewriter",
                    "parameter": {
                        "path": "D:/data/",
                        "fileName": "oracledata",
                        "writeMode": "truncate",
                        "format": "yyyy-MM-dd"
                       }
                   }  
                } 
        ],  
        "setting": {  
            "speed": {  
                "channel": 1  
            }  
        }  
    }  
}  

 

 

2.oracle ->oracle

 

{  
    "job": {  
        "content": [  
            {  
                 "reader": {
                    "name": "oraclereader",
                    "parameter": {
                        "username": "xxxx",
                        "password": "xxxx",
                        "column": ["*"],
                        "where": "",
                        "connection": [
                            {
                                "querySql": [
                                    "select * from table1"
                                ],
                                "jdbcUrl": [
                                    "jdbc:oracle:thin:@192.168.121.30:1521:tcenter"
                                ]
                            }
                        ]
                    }
                },  
                "writer": { 
            		"name": "oraclewriter", 
            		"parameter": {
                        "username": "xxxx",
	                    "password": "xxxx",
	                    "truncate": "true",
                        "column": ["SYSTEMNO",
                                "SYSTEMTIME",
                                "xxx",
                                "x",
                                "x",
                                "x",
                                "x",
                                "x",
                                "x",
                                "x",
                                "x",
                                "x",
                                "STATUS",
                                "x",
                                "x",
                                "x"
                                ],
	                    "connection": [
                            {
                                
                                "jdbcUrl": "jdbc:oracle:thin:@192.168.121.30:1521:tcexxnter",
                                "table": [
                                    "table1"
                                ]
                            }
                        ]
                      }
                }  
            } 
        ],  
        "setting": {  
            "speed": {  
                "channel": 6 
            }  
        }  
    }  
}  

 

 

Relevant Query Input demo:

 

 

Building table SQL:

 

CREATE TABLE ZF_PaytDemo(
       SystemNo          VARCHAR2(32)  NOT NULL,
       MainOrderSystemNo		VARCHAR2(32)	NOT NULL,
       TxAmount          NUMBER(16)    NOT NULL,
       AccountAmount       NUMBER(16)    NOT NULL,
       FinanceChannelCode		VARCHAR2(64)  	NOT NULL
);

Json:

 

 

{  
    "job": {  
        "content": [  
            {  
                 "reader": {
                    "name": "oraclereader",
                    "parameter": {
                        "username": "xxx",
                        "password": "xxx",
                        "where": "",
                        "connection": [
                            {
                                "querySql": [
                                    "select B.SYSTEMNO, A.SYSTEMNO AS MainOrderSystemNo, B.txamount, B.AccountAmount,  B.Financechannelcode from zf_fundinordermain A left join zf_fundinorderpayt B on A.Systemno = B.MAINORDERSYSTEMNO where  A.Systemno ='35112016091411580603700000' "
                                ],
                                "jdbcUrl": [
                                    "jdbc:oracle:thin:@192.168.121.30:1521:tcenter"
                                ]
                            }
                        ]
                    }
                },  
                "writer": { 
            		"name": "oraclewriter", 
            		"parameter": {
                        "username": "xxx",
	                    "password": "xxx",
	                    "truncate": "false",
                        "column": [
                            "SystemNo",
                            "MainOrderSystemNo",
                            "TxAmount",
                            "AccountAmount",
                            "FinanceChannelCode"
                        ],
	                    "connection": [
                            {
                                
                                "jdbcUrl": "jdbc:oracle:thin:@192.168.121.30:1521:tcenter",
                                "table": [
                                    "ZF_PaytDemo"
                                ]
                            }
                        ]
                      }
                }  
           }
        ],  
        "setting": {  
            "speed": {  
                "channel": 1  
            }  
        }  
    }  
}  

 

file->oracle JSON:

 

 

{  
    "job": {  
        "content": [  
            {  
                 "reader": {
                    "name": "txtfilereader",
                    "parameter": {
                        "path": ["D:/datax.txt"],
                        "encoding": "UTF-8",
                        "column": [
                            {
                                "index": 0,
                                "type": "string"
                            },
                            {
                                "index": 1,
                                "type": "string"
                            },
                            {
                                "index": 2,
                                "type": "long"
                            },
                            {
                                "index": 3,
                                "type": "long"
                            },
                            {
                                "index": 4,
                                "type": "string"
                            }
                        ],
                        "fieldDelimiter": "\t"
                    }
                },  
                "writer": { 
            		"name": "oraclewriter", 
            		"parameter": {
                        "username": "xxxx",
	                    "password": "xxxx",
	                    "truncate": "true",
                        "column": [  
                            "SystemNo",
                            "MainOrderSystemNo",
                            "TxAmount",
                            "AccountAmount",
                            "FinanceChannelCode"
                        ],
	                    "connection": [
                            {
                                
                                "jdbcUrl": "jdbc:oracle:thin:@192.168.121.30:1521:tcenter",
                                "table": [
                                    "ZF_PaytDemo"
                                ]
                            }
                        ]
                      }
                }  
            } 
        ],  
        "setting": {  
            "speed": {  
                "channel": 6 
            }  
        }  
    }  
}  

 

 

 

 

 

 

 

 

 

Solve the problem, give everyone benefits, there is a valid period, sweep code, quickly expired.

 

Tags: Oracle JDBC JSON Python

Posted on Tue, 08 Oct 2019 00:14:34 -0700 by programming.name