diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.gitignore b/.gitignore index a32168f1a..cedc8fd14 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ target/ plugins/ lib/ .vertx/ -.DS_Store bin/nohup.out -.DS_Store + bin/sideSql.txt diff --git a/README.md b/README.md index 6474dc052..60f76f621 100644 --- a/README.md +++ b/README.md @@ -7,45 +7,34 @@ > > * 支持原生FLinkSQL所有的语法 > > * 扩展了输入和输出的性能指标到promethus - ## 新特性: - * 1.kafka源表支持not null语法,支持字符串类型的时间转换。 - * 2.rdb维表与DB建立连接时，周期进行连接，防止连接断开。rdbsink写入时，对连接进行检查。 - * 3.异步维表支持非等值连接，比如：<>,<,>。 - * 4.增加kafka数组解析 - * 5.增加kafka1.0以上版本的支持 - * 6.增加postgresql、kudu、clickhouse维表、结果表的支持 - * 7.支持插件的依赖方式,参考pluginLoadMode参数 - * 8.支持cep处理 - * 9.支持udaf - * 10.支持谓词下移 - - ## BUG修复： - * 1.修复不能解析sql中orderby,union语法。 - * 2.修复yarnPer模式提交失败的异常。 - * 3.一些bug的修复 + +## BUG修复： + + 1. oracle维表获取索引问题。 + 2. Perjob模式下UDF类加载异常问题。 + 3. queue 参数设置无效的问题。 + +## 新特性： + 1. 支持从kafka读取嵌套JSON格式数据,暂不支持数组类型字段。例如： info.name varchar as info_name。 + 2. 支持kafka结果表数据写入。 + 3. 支持为ROWTIME绑定时区，默认为本地时区。例如：timezone="America/New_York" + 4. 支持从kafka自定义偏移量中消费数据。 # 已支持 - * 源表：kafka 0.9、0.10、0.11、1.x版本 - * 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, serversocket, kudu, postgresql, clickhouse - * 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse + * 源表：kafka 0.9，1.x版本，serverSocket + * 维表：mysql，SQlServer,oracle,hbase，mongo，redis,cassandra + * 结果表：mysql，SQlServer,oracle,hbase，elasticsearch5.x，mongo，redis,cassandra，console # 后续开发计划 * 维表快照 + * sql优化（谓词下移等） * kafka avro格式 * topN -## 1 快速起步 -### 1.1 运行模式 - - -* 单机模式：对应Flink集群的单机模式 -* standalone模式：对应Flink集群的分布式模式 -* yarn模式：对应Flink集群的yarn模式 - ### 1.2 执行环境 * Java: JDK8及以上 -* Flink集群: 1.4,1.5,1.8（单机模式不需要安装Flink集群） +* Flink集群: 1.4,1.5（单机模式不需要安装Flink集群） * 操作系统：理论上不限 ### 1.3 打包 @@ -63,11 +52,15 @@ mvn clean package -Dmaven.test.skip #### 1.4.1 启动命令 ``` -sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack/150_flinkplugin/sqlplugin -localSqlPluginPath D:\gitspace\flinkStreamSQL\plugins -addjar \["udf.jar\"\] -mode yarn -flinkconf D:\flink_home\kudu150etc -yarnconf D:\hadoop\etc\hadoopkudu -confProp \{\"time.characteristic\":\"EventTime\",\"sql.checkpoint.interval\":10000\} -yarnSessionConf \{\"yid\":\"application_1564971615273_38182\"} +sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack/150_flinkplugin/sqlplugin -localSqlPluginPath D:\gitspace\flinkStreamSQL\plugins -addjar \["udf.jar\"\] -mode yarn -flinkconf D:\flink_home\kudu150etc -yarnconf D:\hadoop\etc\hadoopkudu -confProp \{\"time.characteristic\":\"EventTime\",\"sql.checkpoint.interval\":10000\} ``` #### 1.4.2 命令行参数选项 +* 单机模式：对应Flink集群的单机模式 +* standalone模式：对应Flink集群的分布式模式 +* yarn模式：对应Flink集群的yarn模式 + * **mode** * 描述：执行模式，也就是flink集群的工作模式 * local: 本地模式 @@ -154,16 +147,6 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * 描述：per_job 模式下指定的yarn queue * 必选：否 * 默认值：false - -* **pluginLoadMode** - * 描述：per_job 模式下的插件包加载方式。classpath:从每台机器加载插件包，shipfile:将需要插件从提交的节点上传到hdfs,不需要每台安装插件 - * 必选：否 - * 默认值：classpath - -* **yarnSessionConf** - * 描述：yarn session 模式下指定的运行的一些参数，[可参考](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/cli.html),目前只支持指定yid - * 必选：否 - * 默认值：false ## 2 结构 ### 2.1 源表插件 @@ -173,24 +156,16 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack * [elasticsearch 结果表插件](docs/elasticsearchSink.md) * [hbase 结果表插件](docs/hbaseSink.md) * [mysql 结果表插件](docs/mysqlSink.md) -* [oracle 结果表插件](docs/oracleSink.md) * [mongo 结果表插件](docs/mongoSink.md) * [redis 结果表插件](docs/redisSink.md) * [cassandra 结果表插件](docs/cassandraSink.md) -* [kudu 结果表插件](docs/kuduSink.md) -* [postgresql 结果表插件](docs/postgresqlSink.md) -* [clickhouse 结果表插件](docs/clickhouseSink.md) ### 2.3 维表插件 * [hbase 维表插件](docs/hbaseSide.md) * [mysql 维表插件](docs/mysqlSide.md) -* [oracle 维表插件](docs/oracleSide.md) * [mongo 维表插件](docs/mongoSide.md) * [redis 维表插件](docs/redisSide.md) * [cassandra 维表插件](docs/cassandraSide.md) -* [kudu 维表插件](docs/kuduSide.md) -* [postgresql 维表插件](docs/postgresqlSide.md) -* [clickhouse 维表插件](docs/clickhouseSide.md) ## 3 性能指标(新增) @@ -221,7 +196,7 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack ``` -CREATE (scala|table|aggregate) FUNCTION CHARACTER_LENGTH WITH com.dtstack.Kun; +CREATE (scala|table) FUNCTION CHARACTER_LENGTH WITH com.dtstack.Kun; CREATE TABLE MyTable( @@ -300,4 +275,3 @@ into # 招聘 1.大数据平台开发工程师，想了解岗位详细信息可以添加本人微信号ysqwhiletrue,注明招聘，如有意者发送简历至sishu@dtstack.com。 - diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java index eecb1350e..3a7e56902 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllReqRow.java @@ -36,8 +36,8 @@ import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Maps; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; diff --git a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java index fa665f9a0..bba39fc3e 100644 --- a/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java +++ b/cassandra/cassandra-side/cassandra-all-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAllSideInfo.java @@ -23,11 +23,12 @@ import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; -import com.dtstack.flink.sql.util.ParseUtils; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import java.util.List; @@ -52,6 +53,7 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { sqlCondition = "select ${selectField} from ${tableName} "; sqlCondition = sqlCondition.replace("${tableName}", cassandraSideTableInfo.getDatabase() + "." + cassandraSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); + System.out.println("---------side_exe_sql-----\n" + sqlCondition); } @Override @@ -84,7 +86,11 @@ public void parseSelectFields(JoinInfo joinInfo) { SqlNode conditionNode = joinInfo.getCondition(); List sqlNodeList = Lists.newArrayList(); - ParseUtils.parseAnd(conditionNode, sqlNodeList); + if (conditionNode.getKind() == SqlKind.AND) { + sqlNodeList.addAll(Lists.newArrayList(((SqlBasicCall) conditionNode).getOperands())); + } else { + sqlNodeList.add(conditionNode); + } for (SqlNode sqlNode : sqlNodeList) { dealOneEqualCon(sqlNode, sideTableName); diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java index c2528d67b..94c8e6fb6 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java @@ -45,7 +45,7 @@ import io.vertx.core.json.JsonArray; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; -import com.google.common.collect.Lists; +import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -256,7 +256,7 @@ public void onFailure(Throwable t) { t.getMessage()); System.out.println("Failed to retrieve the data: " + t.getMessage()); cluster.closeAsync(); - resultFuture.completeExceptionally(t); + resultFuture.complete(null); } }); } diff --git a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java index 3557f0f73..b1d239440 100644 --- a/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java +++ b/cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncSideInfo.java @@ -23,13 +23,12 @@ import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.cassandra.table.CassandraSideTableInfo; -import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import java.util.List; @@ -56,7 +55,11 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { SqlNode conditionNode = joinInfo.getCondition(); List sqlNodeList = Lists.newArrayList(); - ParseUtils.parseAnd(conditionNode, sqlNodeList); + if (conditionNode.getKind() == SqlKind.AND) { + sqlNodeList.addAll(Lists.newArrayList(((SqlBasicCall) conditionNode).getOperands())); + } else { + sqlNodeList.add(conditionNode); + } for (SqlNode sqlNode : sqlNodeList) { dealOneEqualCon(sqlNode, sideTableName); diff --git a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideTableInfo.java b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideTableInfo.java index 205a0f769..b1b36f7e8 100644 --- a/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideTableInfo.java +++ b/cassandra/cassandra-side/cassandra-side-core/src/main/java/com/dtstack/flink/sql/side/cassandra/table/CassandraSideTableInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.cassandra.table; import com.dtstack.flink.sql.side.SideTableInfo; -import com.google.common.base.Preconditions; +import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; /** * Reason: diff --git a/cassandra/cassandra-sink/pom.xml b/cassandra/cassandra-sink/pom.xml index 0d9b10361..4fb20c373 100644 --- a/cassandra/cassandra-sink/pom.xml +++ b/cassandra/cassandra-sink/pom.xml @@ -71,7 +71,7 @@ + tofile="${basedir}/../../plugins/cassandrasink/${project.name}.jar" /> diff --git a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java index c6626c42a..7d52b23bb 100644 --- a/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java +++ b/cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/table/CassandraTableInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.cassandra.table; import com.dtstack.flink.sql.table.TargetTableInfo; -import com.google.common.base.Preconditions; +import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; /** * Reason: diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/pom.xml b/clickhouse/clickhouse-side/clickhouse-all-side/pom.xml deleted file mode 100644 index 8e6c6f51a..000000000 --- a/clickhouse/clickhouse-side/clickhouse-all-side/pom.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - - sql.side.clickhouse - com.dtstack.flink - 1.0-SNAPSHOT - - 4.0.0 - - sql.side.all.clickhouse - clickhouse-all-side - - jar - - - 1.0-SNAPSHOT - - - - - com.dtstack.flink - sql.side.clickhouse.core - ${sql.side.clickhouse.core.version} - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 1.4 - - - package - - shade - - - - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - maven-antrun-plugin - 1.2 - - - copy-resources - - package - - run - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java deleted file mode 100644 index 68c0c7984..000000000 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.side.clickhouse; - -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.JDBCUtils; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.util.List; -import java.util.Map; - -public class ClickhouseAllReqRow extends RdbAllReqRow { - - private static final Logger LOG = LoggerFactory.getLogger(ClickhouseAllReqRow.class); - - private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - - public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(new ClickhouseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); - } - - @Override - public Connection getConn(String dbURL, String userName, String passWord) { - try { - Connection connection ; - JDBCUtils.forName(CLICKHOUSE_DRIVER, getClass().getClassLoader()); - // ClickHouseProperties contains all properties - if (userName == null) { - connection = DriverManager.getConnection(dbURL); - } else { - connection = DriverManager.getConnection(dbURL, userName, passWord); - } - return connection; - } catch (Exception e) { - LOG.error("", e); - throw new RuntimeException("", e); - } - } - -} diff --git a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java b/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java deleted file mode 100644 index 973c069b9..000000000 --- a/clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.side.clickhouse; - -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; -import org.apache.flink.api.java.typeutils.RowTypeInfo; - -import java.util.List; - - -public class ClickhouseAllSideInfo extends RdbAllSideInfo { - public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); - } -} diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/pom.xml b/clickhouse/clickhouse-side/clickhouse-async-side/pom.xml deleted file mode 100644 index ab889c5a8..000000000 --- a/clickhouse/clickhouse-side/clickhouse-async-side/pom.xml +++ /dev/null @@ -1,91 +0,0 @@ - - - - sql.side.clickhouse - com.dtstack.flink - 1.0-SNAPSHOT - - 4.0.0 - - sql.side.async.clickhouse - clickhouse-async-side - - jar - - - 1.0-SNAPSHOT - - - - - com.dtstack.flink - sql.side.clickhouse.core - ${sql.side.clickhouse.core.version} - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 1.4 - - - package - - shade - - - - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - maven-antrun-plugin - 1.2 - - - copy-resources - - package - - run - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java deleted file mode 100644 index 305d65118..000000000 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncReqRow.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.side.clickhouse; - -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; -import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import io.vertx.core.Vertx; -import io.vertx.core.VertxOptions; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.jdbc.JDBCClient; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; - -import java.util.List; - - -public class ClickhouseAsyncReqRow extends RdbAsyncReqRow { - private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - - public ClickhouseAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(new ClickhouseAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); - } - - @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - JsonObject clickhouseClientConfig = new JsonObject(); - RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - clickhouseClientConfig.put("url", rdbSideTableInfo.getUrl()) - .put("driver_class", CLICKHOUSE_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) - .put("user", rdbSideTableInfo.getUserName()) - .put("password", rdbSideTableInfo.getPassword()) - .put("provider_class", DT_PROVIDER_CLASS); - System.setProperty("vertx.disableFileCPResolving", "true"); - VertxOptions vo = new VertxOptions(); - vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); - vo.setFileResolverCachingEnabled(false); - Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, clickhouseClientConfig)); - } - -} diff --git a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java b/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java deleted file mode 100644 index 254561de0..000000000 --- a/clickhouse/clickhouse-side/clickhouse-async-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAsyncSideInfo.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.side.clickhouse; - -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; -import org.apache.flink.api.java.typeutils.RowTypeInfo; - -import java.util.List; - - -public class ClickhouseAsyncSideInfo extends RdbAsyncSideInfo { - - public ClickhouseAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); - } -} diff --git a/clickhouse/clickhouse-side/clickhouse-side-core/pom.xml b/clickhouse/clickhouse-side/clickhouse-side-core/pom.xml deleted file mode 100644 index 29304d922..000000000 --- a/clickhouse/clickhouse-side/clickhouse-side-core/pom.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - sql.side.clickhouse - com.dtstack.flink - 1.0-SNAPSHOT - - 4.0.0 - - sql.side.clickhouse.core - 1.0-SNAPSHOT - jar - clickhouse-side-core - - \ No newline at end of file diff --git a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java b/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java deleted file mode 100644 index 7be387fd8..000000000 --- a/clickhouse/clickhouse-side/clickhouse-side-core/src/main/java/com/dtstack/flink/sql/side/clickhouse/table/ClickhouseSideParser.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.side.clickhouse.table; - -import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; -import ru.yandex.clickhouse.domain.ClickHouseDataType; - -import java.util.Map; - -/** - * Reason: - * Date: 2019/11/04 - * Company: www.dtstack.com - * - * @author maqi - */ - -public class ClickhouseSideParser extends RdbSideParser { - - private static final String CURR_TYPE = "clickhouse"; - - @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); - clickhouseTableInfo.setType(CURR_TYPE); - return clickhouseTableInfo; - } - - @Override - public Class dbTypeConvertToJavaType(String fieldType) { - return ClickHouseDataType.fromTypeString(fieldType).getJavaClass(); - } - -} diff --git a/clickhouse/clickhouse-side/pom.xml b/clickhouse/clickhouse-side/pom.xml deleted file mode 100644 index 2ba6a14dc..000000000 --- a/clickhouse/clickhouse-side/pom.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - sql.clickhouse - com.dtstack.flink - 1.0-SNAPSHOT - - 4.0.0 - - sql.side.clickhouse - 1.0-SNAPSHOT - clickhouse-side - pom - - - clickhouse-side-core - clickhouse-async-side - clickhouse-all-side - - - - - 1.0-SNAPSHOT - - - - - com.dtstack.flink - sql.side.rdb - ${rdb.side.version} - - - - \ No newline at end of file diff --git a/clickhouse/clickhouse-sink/pom.xml b/clickhouse/clickhouse-sink/pom.xml deleted file mode 100644 index 75aac514a..000000000 --- a/clickhouse/clickhouse-sink/pom.xml +++ /dev/null @@ -1,90 +0,0 @@ - - - - sql.clickhouse - com.dtstack.flink - 1.0-SNAPSHOT - - 4.0.0 - - sql.sink.clickhouse - clickhouse-sink - http://maven.apache.org - - - 1.0-SNAPSHOT - - - - - com.dtstack.flink - sql.sink.rdb - ${sql.sink.rdb.version} - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 1.4 - - - package - - shade - - - - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - maven-antrun-plugin - 1.2 - - - copy-resources - - package - - run - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java deleted file mode 100644 index 66c1f26d5..000000000 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/ClickhouseSink.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.sink.clickhouse; - - -import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; - -import java.util.List; -import java.util.Map; - - -public class ClickhouseSink extends RdbSink implements IStreamSinkGener { - - private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver"; - - public ClickhouseSink() { - } - - @Override - public RetractJDBCOutputFormat getOutputFormat() { - return new RetractJDBCOutputFormat(); - } - - @Override - public void buildSql(String scheam, String tableName, List fields) { - buildInsertSql(tableName, fields); - } - - @Override - public String buildUpdateSql(String schema, String tableName, List fieldNames, Map> realIndexes, List fullField) { - return null; - } - - private void buildInsertSql(String tableName, List fields) { - String sqlTmp = "insert into " + tableName + " (${fields}) values (${placeholder})"; - String fieldsStr = ""; - String placeholder = ""; - - for (String fieldName : fields) { - fieldsStr += ",`" + fieldName + "`"; - placeholder += ",?"; - } - - fieldsStr = fieldsStr.replaceFirst(",", ""); - placeholder = placeholder.replaceFirst(",", ""); - - sqlTmp = sqlTmp.replace("${fields}", fieldsStr).replace("${placeholder}", placeholder); - this.sql = sqlTmp; - System.out.println("---insert sql----"); - System.out.println(sql); - } - - - @Override - public String getDriverName() { - return CLICKHOUSE_DRIVER; - } - - -} diff --git a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java b/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java deleted file mode 100644 index 8c3df93d7..000000000 --- a/clickhouse/clickhouse-sink/src/main/java/com/dtstack/flink/sql/sink/clickhouse/table/ClickhouseSinkParser.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.sink.clickhouse.table; - -import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; -import ru.yandex.clickhouse.domain.ClickHouseDataType; - -import java.util.Map; - - -public class ClickhouseSinkParser extends RdbSinkParser { - private static final String CURR_TYPE = "clickhouse"; - - @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo clickhouseTableInfo = super.getTableInfo(tableName, fieldsInfo, props); - clickhouseTableInfo.setType(CURR_TYPE); - return clickhouseTableInfo; - } - - @Override - public Class dbTypeConvertToJavaType(String fieldType) { - return ClickHouseDataType.fromTypeString(fieldType).getJavaClass(); - } - -} \ No newline at end of file diff --git a/clickhouse/pom.xml b/clickhouse/pom.xml deleted file mode 100644 index 37589bb36..000000000 --- a/clickhouse/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - flink.sql - com.dtstack.flink - 1.0-SNAPSHOT - - 4.0.0 - - sql.clickhouse - pom - - - clickhouse-side - clickhouse-sink - - - - - 1.0-SNAPSHOT - 0.1.55 - - - - - com.dtstack.flink - sql.core - ${sql.core.version} - provided - - - - ru.yandex.clickhouse - clickhouse-jdbc - ${clickhouse.jdbc.version} - - - - \ No newline at end of file diff --git a/console/console-sink/pom.xml b/console/console-sink/pom.xml deleted file mode 100644 index 2f8ad9ef9..000000000 --- a/console/console-sink/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - sql.console - com.dtstack.flink - 1.0-SNAPSHOT - - 4.0.0 - - console-sink - jar - - console-sink - http://maven.apache.org - - - - org.apache.maven.plugins - maven-shade-plugin - 1.4 - - - package - - shade - - - - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - maven-antrun-plugin - 1.2 - - - copy-resources - - package - - run - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java deleted file mode 100644 index 7658e9979..000000000 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleOutputFormat.java +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.console; - -import com.dtstack.flink.sql.sink.MetricOutputFormat; -import com.dtstack.flink.sql.sink.console.table.TablePrintUtil; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.types.Row; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Reason: - * Date: 2018/12/19 - * - * @author xuqianjin - */ -public class ConsoleOutputFormat extends MetricOutputFormat { - - private static final Logger LOG = LoggerFactory.getLogger(ConsoleOutputFormat.class); - - protected String[] fieldNames; - TypeInformation[] fieldTypes; - - @Override - public void configure(Configuration parameters) { - - } - - @Override - public void open(int taskNumber, int numTasks) throws IOException { - initMetric(); - } - - @Override - public void writeRecord(Tuple2 tuple2) throws IOException { - Tuple2 tupleTrans = tuple2; - Boolean retract = tupleTrans.getField(0); - if (!retract) { - return; - } - - Row record = tupleTrans.getField(1); - if (record.getArity() != fieldNames.length) { - return; - } - - List data = new ArrayList<>(); - data.add(fieldNames); - data.add(record.toString().split(",")); - TablePrintUtil.build(data).print(); - - outRecords.inc(); - } - - @Override - public void close() throws IOException { - - } - - private ConsoleOutputFormat() { - } - - public static ConsoleOutputFormatBuilder buildOutputFormat() { - return new ConsoleOutputFormatBuilder(); - } - - public static class ConsoleOutputFormatBuilder { - - private final ConsoleOutputFormat format; - - protected ConsoleOutputFormatBuilder() { - this.format = new ConsoleOutputFormat(); - } - - public ConsoleOutputFormatBuilder setFieldNames(String[] fieldNames) { - format.fieldNames = fieldNames; - return this; - } - - public ConsoleOutputFormatBuilder setFieldTypes(TypeInformation[] fieldTypes) { - format.fieldTypes = fieldTypes; - return this; - } - - /** - * Finalizes the configuration and checks validity. - * - * @return Configured RetractConsoleCOutputFormat - */ - public ConsoleOutputFormat finish() { - return format; - } - } -} diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java deleted file mode 100644 index 77a3efea2..000000000 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/ConsoleSink.java +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.console; - -import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.functions.sink.OutputFormatSinkFunction; -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.types.Row; - -/** - * Reason: - * Date: 2018/12/19 - * - * @author xuqianjin - */ -public class ConsoleSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - TypeInformation[] fieldTypes; - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), getRecordType()); - } - - @Override - public TypeInformation getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public void emitDataStream(DataStream> dataStream) { - ConsoleOutputFormat.ConsoleOutputFormatBuilder builder = ConsoleOutputFormat.buildOutputFormat(); - builder.setFieldNames(this.fieldNames) - .setFieldTypes(this.fieldTypes); - ConsoleOutputFormat outputFormat = builder.finish(); - RichSinkFunction richSinkFunction = new OutputFormatSinkFunction(outputFormat); - dataStream.addSink(richSinkFunction); - } - - @Override - public ConsoleSink genStreamSink(TargetTableInfo targetTableInfo) { - return this; - } -} diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java deleted file mode 100644 index e77444bfd..000000000 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleSinkParser.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.console.table; - -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.util.MathUtil; - -import java.util.Map; - -import static com.dtstack.flink.sql.table.TableInfo.PARALLELISM_KEY; - -/** - * Reason: - * Date: 2018/12/19 - * - * @author xuqianjin - */ -public class ConsoleSinkParser extends AbsTableParser { - @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - ConsoleTableInfo consoleTableInfo = new ConsoleTableInfo(); - consoleTableInfo.setName(tableName); - parseFieldsInfo(fieldsInfo, consoleTableInfo); - - consoleTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(PARALLELISM_KEY.toLowerCase()))); - return consoleTableInfo; - } -} diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java deleted file mode 100644 index 4b286c667..000000000 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/ConsoleTableInfo.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.console.table; - -import com.dtstack.flink.sql.table.TargetTableInfo; - -/** - * Reason: - * Date: 2018/12/19 - * - * @author xuqianjin - */ -public class ConsoleTableInfo extends TargetTableInfo { - - private static final String CURR_TYPE = "console"; - - public ConsoleTableInfo() { - setType(CURR_TYPE); - } - - @Override - public boolean check() { - return true; - } - - @Override - public String getType() { - return super.getType().toLowerCase(); - } -} diff --git a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java b/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java deleted file mode 100644 index 8813da619..000000000 --- a/console/console-sink/src/main/java/com/dtstack/flink/sql/sink/console/table/TablePrintUtil.java +++ /dev/null @@ -1,341 +0,0 @@ -package com.dtstack.flink.sql.sink.console.table; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Reason: - * Date: 2018/12/19 - * - * @author xuqianjin - */ -public class TablePrintUtil { - private static final Logger LOG = LoggerFactory.getLogger(TablePrintUtil.class); - public static final int ALIGN_LEFT = 1;//左对齐 - public static final int ALIGN_RIGHT = 2;//右对齐 - public static final int ALIGN_CENTER = 3;//居中对齐 - - private int align = ALIGN_CENTER;//默认居中对齐 - private boolean equilong = false;//默认不等宽 - private int padding = 1;//左右边距默认为1 - private char h = '-';//默认水平分隔符 - private char v = '|';//默认竖直分隔符 - private char o = '+';//默认交叉分隔符 - private char s = ' ';//默认空白填充符 - private List data;//数据 - - private TablePrintUtil() { - } - - /** - * 链式调用入口方法 - * - * @param data - * @return - */ - public static TablePrintUtil build(String[][] data) { - TablePrintUtil self = new TablePrintUtil(); - self.data = new ArrayList<>(Arrays.asList(data)); - return self; - } - - /** - * 链式调用入口方法，T可以是String[]、List、任意实体类 - * 由于java泛型不同无法重载，所以这里要写if instanceof进行类型判断 - * - * @param data - * @param - * @return - */ - public static TablePrintUtil build(List data) { - TablePrintUtil self = new TablePrintUtil(); - self.data = new ArrayList<>(); - if (data.size() <= 0) throw new RuntimeException("数据源至少得有一行吧"); - Object obj = data.get(0); - - - if (obj instanceof String[]) { - //如果泛型为String数组，则直接设置 - self.data = (List) data; - } else if (obj instanceof List) { - //如果泛型为List，则把list中的item依次转为String[]，再设置 - int length = ((List) obj).size(); - for (Object item : data) { - List col = (List) item; - if (col.size() != length) throw new RuntimeException("数据源每列长度必须一致"); - self.data.add(col.toArray(new String[length])); - } - } else { - //如果泛型为实体类，则利用反射获取get方法列表，从而推算出属性列表。 - //根据反射得来的属性列表设置表格第一行thead - List colList = getColList(obj); - String[] header = new String[colList.size()]; - for (int i = 0; i < colList.size(); i++) { - header[i] = colList.get(i).colName; - } - self.data.add(header); - //利用反射调用相应get方法获取属性值来设置表格tbody - for (int i = 0; i < data.size(); i++) { - String[] item = new String[colList.size()]; - for (int j = 0; j < colList.size(); j++) { - String value = null; - try { - value = obj.getClass().getMethod(colList.get(j).getMethodName).invoke(data.get(i)).toString(); - } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { - e.printStackTrace(); - } - item[j] = value == null ? "null" : value; - } - self.data.add(item); - } - } - return self; - } - - private static class Col { - private String colName;//列名 - private String getMethodName;//get方法名 - } - - /** - * 利用反射获取get方法名和属性名 - * - * @return - */ - private static List getColList(Object obj) { - List colList = new ArrayList<>(); - Method[] methods = obj.getClass().getMethods(); - for (Method m : methods) { - StringBuilder getMethodName = new StringBuilder(m.getName()); - if (getMethodName.substring(0, 3).equals("get") && !m.getName().equals("getClass")) { - Col col = new Col(); - col.getMethodName = getMethodName.toString(); - char first = Character.toLowerCase(getMethodName.delete(0, 3).charAt(0)); - getMethodName.delete(0, 1).insert(0, first); - col.colName = getMethodName.toString(); - colList.add(col); - } - } - return colList; - } - - /** - * 获取字符串占的字符位数 - * - * @param str - * @return - */ - private int getStringCharLength(String str) { - Pattern p = Pattern.compile("[\u4e00-\u9fa5]");//利用正则找到中文 - Matcher m = p.matcher(str); - int count = 0; - while (m.find()) { - count++; - } - return str.length() + count; - } - - /** - * 纵向遍历获取数据每列的长度 - * - * @return - */ - private int[] getColLengths() { - int[] result = new int[data.get(0).length]; - for (int x = 0; x < result.length; x++) { - int max = 0; - for (int y = 0; y < data.size(); y++) { - int len = getStringCharLength(data.get(y)[x]); - if (len > max) { - max = len; - } - } - result[x] = max; - } - if (equilong) {//如果等宽表格 - int max = 0; - for (int len : result) { - if (len > max) max = len; - } - for (int i = 0; i < result.length; i++) { - result[i] = max; - } - } - return result; - } - - /** - * 取得表格字符串 - * - * @return - */ - public String getTableString() { - StringBuilder sb = new StringBuilder(); - int[] colLengths = getColLengths();//获取每列文字宽度 - StringBuilder line = new StringBuilder();//表格横向分隔线 - line.append(o); - for (int len : colLengths) { - int allLen = len + padding * 2;//还需要加上边距和分隔符的长度 - for (int i = 0; i < allLen; i++) { - line.append(h); - } - line.append(o); - } - sb.append(line).append("\r\n"); - for (int y = 0; y < data.size(); y++) { - sb.append(v); - for (int x = 0; x < data.get(y).length; x++) { - String cell = data.get(y)[x]; - switch (align) { - case ALIGN_LEFT: - for (int i = 0; i < padding; i++) {sb.append(s);} - sb.append(cell); - for (int i = 0; i < colLengths[x] - getStringCharLength(cell) + padding; i++) {sb.append(s);} - break; - case ALIGN_RIGHT: - for (int i = 0; i < colLengths[x] - getStringCharLength(cell) + padding; i++) {sb.append(s);} - sb.append(cell); - for (int i = 0; i < padding; i++) {sb.append(s);} - break; - case ALIGN_CENTER: - int space = colLengths[x] - getStringCharLength(cell); - int left = space / 2; - int right = space - left; - for (int i = 0; i < left + padding; i++) {sb.append(s);} - sb.append(cell); - for (int i = 0; i < right + padding; i++) {sb.append(s);} - break; - } - sb.append(v); - } - sb.append("\r\n"); - sb.append(line).append("\r\n"); - } - return sb.toString(); - } - - /** - * 直接打印表格 - */ - public void print() { - LOG.info("\n"+getTableString()); - System.out.println(getTableString()); - } - - //下面是链式调用的set方法 - public TablePrintUtil setAlign(int align) { - this.align = align; - return this; - } - - public TablePrintUtil setEquilong(boolean equilong) { - this.equilong = equilong; - return this; - } - - public TablePrintUtil setPadding(int padding) { - this.padding = padding; - return this; - } - - public TablePrintUtil setH(char h) { - this.h = h; - return this; - } - - public TablePrintUtil setV(char v) { - this.v = v; - return this; - } - - public TablePrintUtil setO(char o) { - this.o = o; - return this; - } - - public TablePrintUtil setS(char s) { - this.s = s; - return this; - } - - /** - * 使用示例 - * - * @param args - */ - public static void main(String[] args) { - List data1 = new ArrayList<>(); - data1.add(new String[]{"用户名", "密码", "姓名"}); - data1.add(new String[]{"xiaoming", "xm123", "小明"}); - data1.add(new String[]{"xiaohong", "xh123", "小红"}); - TablePrintUtil.build(data1).print(); - - List> data2 = new ArrayList<>(); - data2.add(new ArrayList<>()); - data2.add(new ArrayList<>()); - data2.add(new ArrayList<>()); - data2.get(0).add("用户名"); - data2.get(0).add("密码"); - data2.get(0).add("姓名"); - data2.get(1).add("xiaoming"); - data2.get(1).add("xm123"); - data2.get(1).add("小明"); - data2.get(2).add("xiaohong"); - data2.get(2).add("xh123"); - data2.get(2).add("小红"); - TablePrintUtil.build(data2) - .setAlign(TablePrintUtil.ALIGN_LEFT) - .setPadding(5) - .setEquilong(true) - .print(); - - - class User { - String username; - String password; - String name; - - User(String username, String password, String name) { - this.username = username; - this.password = password; - this.name = name; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - } - List data3 = new ArrayList<>(); - data3.add(new User("xiaoming", "xm123", "小明")); - data3.add(new User("xiaohong", "xh123", "小红")); - TablePrintUtil.build(data3).setH('=').setV('!').print(); - } -} diff --git a/console/console-sink/src/test/java/com/dtstack/flinkx/AppTest.java b/console/console-sink/src/test/java/com/dtstack/flinkx/AppTest.java deleted file mode 100644 index e03e5451f..000000000 --- a/console/console-sink/src/test/java/com/dtstack/flinkx/AppTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flinkx; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -/** - * Unit test for simple App. - */ -public class AppTest - extends TestCase -{ - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest(String testName ) - { - super( testName ); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() - { - return new TestSuite( AppTest.class ); - } - - /** - * Rigourous Test :-) - */ - public void testApp() - { - assertTrue( true ); - } -} diff --git a/console/pom.xml b/console/pom.xml deleted file mode 100644 index 983e1c185..000000000 --- a/console/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - flink.sql - com.dtstack.flink - 1.0-SNAPSHOT - - 4.0.0 - sql.console - pom - - - console-sink - - - - - junit - junit - 3.8.1 - test - - - com.dtstack.flink - sql.core - 1.0-SNAPSHOT - provided - - - - - \ No newline at end of file diff --git a/core/pom.xml b/core/pom.xml index c357b4c99..cbcc8eaae 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -17,7 +17,6 @@ UTF-8 core - 1.16.0 @@ -28,58 +27,21 @@ test - - joda-time - joda-time - 2.5 - - - - org.apache.flink - flink-core - ${flink.version} - - - - org.apache.flink - flink-streaming-java_2.11 - ${flink.version} - - - org.apache.flink - flink-streaming-scala_2.11 - ${flink.version} - - - - - - org.apache.flink - flink-table-planner_2.11 ${flink.version} org.apache.flink - flink-table-common + flink-core ${flink.version} - - org.apache.calcite - calcite-server - - ${calcite.server.version} - - org.apache.flink - flink-cep-scala_2.11 + flink-streaming-java_2.11 ${flink.version} @@ -91,7 +53,7 @@ org.apache.flink - flink-yarn_2.11 + flink-streaming-scala_2.11 ${flink.version} diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/ClusterMode.java b/core/src/main/java/com/dtstack/flink/sql/ClusterMode.java similarity index 91% rename from core/src/main/java/com/dtstack/flink/sql/enums/ClusterMode.java rename to core/src/main/java/com/dtstack/flink/sql/ClusterMode.java index 341258a43..024a31854 100644 --- a/core/src/main/java/com/dtstack/flink/sql/enums/ClusterMode.java +++ b/core/src/main/java/com/dtstack/flink/sql/ClusterMode.java @@ -17,7 +17,7 @@ * limitations under the License. */ -package com.dtstack.flink.sql.enums; +package com.dtstack.flink.sql; /** * Created by sishu.yss on 2018/10/10. @@ -31,8 +31,4 @@ public enum ClusterMode { ClusterMode(int type){ this.type = type; } - - public int getType(){ - return this.type; - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/Main.java b/core/src/main/java/com/dtstack/flink/sql/Main.java index a08df9b05..533b9a6d4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/Main.java +++ b/core/src/main/java/com/dtstack/flink/sql/Main.java @@ -20,20 +20,9 @@ package com.dtstack.flink.sql; -import com.dtstack.flink.sql.config.CalciteConfig; -import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.constrant.ConfigConstrant; -import com.dtstack.flink.sql.enums.ClusterMode; +import com.dtstack.flink.sql.classloader.DtClassLoader; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.enums.EPluginLoadMode; -import com.dtstack.flink.sql.environment.MyLocalStreamEnvironment; -import com.dtstack.flink.sql.exec.FlinkSQLExec; -import com.dtstack.flink.sql.option.OptionParser; -import com.dtstack.flink.sql.parser.CreateFuncParser; -import com.dtstack.flink.sql.parser.CreateTmpTableParser; -import com.dtstack.flink.sql.parser.InsertSqlParser; -import com.dtstack.flink.sql.parser.SqlParser; -import com.dtstack.flink.sql.parser.SqlTree; +import com.dtstack.flink.sql.parser.*; import com.dtstack.flink.sql.side.SideSqlExec; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.table.SourceTableInfo; @@ -42,27 +31,30 @@ import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PropertiesUtils; import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner; import com.dtstack.flink.sql.util.FlinkUtil; import com.dtstack.flink.sql.util.PluginUtil; +import org.apache.calcite.config.Lex; import org.apache.calcite.sql.SqlInsert; import org.apache.calcite.sql.SqlNode; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Options; import org.apache.commons.io.Charsets; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.time.Time; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; +import org.apache.flink.calcite.shaded.com.google.common.base.Strings; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.collect.Sets; import org.apache.flink.client.program.ContextEnvironment; import org.apache.flink.configuration.Configuration; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamContextEnvironment; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -72,19 +64,22 @@ import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.File; +import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; -import com.dtstack.flink.sql.option.Options; /** * Date: 2018/6/26 @@ -100,19 +95,44 @@ public class Main { private static final Logger LOG = LoggerFactory.getLogger(Main.class); + private static final int failureRate = 3; + + private static final int failureInterval = 6; //min + + private static final int delayInterval = 10; //sec + + private static org.apache.calcite.sql.parser.SqlParser.Config config = org.apache.calcite.sql.parser.SqlParser + .configBuilder() + .setLex(Lex.MYSQL) + .build(); public static void main(String[] args) throws Exception { - OptionParser optionParser = new OptionParser(args); - Options options = optionParser.getOptions(); - String sql = options.getSql(); - String name = options.getName(); - String addJarListStr = options.getAddjar(); - String localSqlPluginPath = options.getLocalSqlPluginPath(); - String remoteSqlPluginPath = options.getRemoteSqlPluginPath(); - String pluginLoadMode = options.getPluginLoadMode(); - String deployMode = options.getMode(); - String confProp = options.getConfProp(); + Options options = new Options(); + options.addOption("sql", true, "sql config"); + options.addOption("name", true, "job name"); + options.addOption("addjar", true, "add jar"); + options.addOption("localSqlPluginPath", true, "local sql plugin path"); + options.addOption("remoteSqlPluginPath", true, "remote sql plugin path"); + options.addOption("confProp", true, "env properties"); + options.addOption("mode", true, "deploy mode"); + + options.addOption("savePointPath", true, "Savepoint restore path"); + options.addOption("allowNonRestoredState", true, "Flag indicating whether non restored state is allowed if the savepoint"); + + CommandLineParser parser = new DefaultParser(); + CommandLine cl = parser.parse(options, args); + String sql = cl.getOptionValue("sql"); + String name = cl.getOptionValue("name"); + String addJarListStr = cl.getOptionValue("addjar"); + String localSqlPluginPath = cl.getOptionValue("localSqlPluginPath"); + String remoteSqlPluginPath = cl.getOptionValue("remoteSqlPluginPath"); + String deployMode = cl.getOptionValue("mode"); + String confProp = cl.getOptionValue("confProp"); + + Preconditions.checkNotNull(sql, "parameters of sql is required"); + Preconditions.checkNotNull(name, "parameters of name is required"); + Preconditions.checkNotNull(localSqlPluginPath, "parameters of localSqlPluginPath is required"); sql = URLDecoder.decode(sql, Charsets.UTF_8.name()); SqlParser.setLocalSqlPluginRoot(localSqlPluginPath); @@ -123,6 +143,17 @@ public static void main(String[] args) throws Exception { addJarFileList = objMapper.readValue(addJarListStr, List.class); } + ClassLoader threadClassLoader = Thread.currentThread().getContextClassLoader(); + DtClassLoader dtClassLoader = new DtClassLoader(new URL[]{}, threadClassLoader); + Thread.currentThread().setContextClassLoader(dtClassLoader); + + URLClassLoader parentClassloader; + if(!ClusterMode.local.name().equals(deployMode)){ + parentClassloader = (URLClassLoader) threadClassLoader.getParent(); + }else{ + parentClassloader = dtClassLoader; + } + confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); StreamExecutionEnvironment env = getStreamExeEnv(confProperties, deployMode); @@ -141,22 +172,13 @@ public static void main(String[] args) throws Exception { Map registerTableCache = Maps.newHashMap(); //register udf - registerUDF(sqlTree, jarURList, tableEnv); + registerUDF(sqlTree, jarURList, parentClassloader, tableEnv); //register table schema - registerTable(sqlTree, env, tableEnv, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode, sideTableMap, registerTableCache); - - sqlTranslation(localSqlPluginPath, tableEnv,sqlTree,sideTableMap,registerTableCache); - - if(env instanceof MyLocalStreamEnvironment) { - ((MyLocalStreamEnvironment) env).setClasspaths(ClassLoaderManager.getClassPath()); - } + registerTable(sqlTree, env, tableEnv, localSqlPluginPath, remoteSqlPluginPath, sideTableMap, registerTableCache); - env.execute(name); - } - - private static void sqlTranslation(String localSqlPluginPath, StreamTableEnvironment tableEnv,SqlTree sqlTree,Map sideTableMap,Map registerTableCache) throws Exception { SideSqlExec sideSqlExec = new SideSqlExec(); sideSqlExec.setLocalSqlPluginPath(localSqlPluginPath); + for (CreateTmpTableParser.SqlParserResult result : sqlTree.getTmpSqlList()) { sideSqlExec.registerTmpTable(result, sideTableMap, tableEnv, registerTableCache); } @@ -165,13 +187,14 @@ private static void sqlTranslation(String localSqlPluginPath, StreamTableEnviron if(LOG.isInfoEnabled()){ LOG.info("exe-sql:\n" + result.getExecSql()); } + boolean isSide = false; + for (String tableName : result.getTargetTableList()) { if (sqlTree.getTmpTableMap().containsKey(tableName)) { CreateTmpTableParser.SqlParserResult tmp = sqlTree.getTmpTableMap().get(tableName); String realSql = DtStringUtil.replaceIgnoreQuota(result.getExecSql(), "`", ""); - - SqlNode sqlNode = org.apache.calcite.sql.parser.SqlParser.create(realSql, CalciteConfig.MYSQL_LEX_CONFIG).parseStmt(); + SqlNode sqlNode = org.apache.calcite.sql.parser.SqlParser.create(realSql,config).parseStmt(); String tmpSql = ((SqlInsert) sqlNode).getSource().toString(); tmp.setExecSql(tmpSql); sideSqlExec.registerTmpTable(tmp, sideTableMap, tableEnv, registerTableCache); @@ -182,21 +205,26 @@ private static void sqlTranslation(String localSqlPluginPath, StreamTableEnviron break; } } + if(isSide){ //sql-dimensional table contains the dimension table of execution sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache); }else{ - FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql()); - if(LOG.isInfoEnabled()){ - LOG.info("exec sql: " + result.getExecSql()); - } + tableEnv.sqlUpdate(result.getExecSql()); } } } } + if(env instanceof MyLocalStreamEnvironment) { + List urlList = new ArrayList<>(); + urlList.addAll(Arrays.asList(dtClassLoader.getURLs())); + ((MyLocalStreamEnvironment) env).setClasspaths(urlList); + } + env.execute(name); } + /** * This part is just to add classpath for the jar when reading remote execution, and will not submit jar from a local * @param env @@ -215,25 +243,27 @@ private static void addEnvClassPath(StreamExecutionEnvironment env, Set cla } } - private static void registerUDF(SqlTree sqlTree, List jarURList, StreamTableEnvironment tableEnv) - throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + private static void registerUDF(SqlTree sqlTree, List jarURList, URLClassLoader parentClassloader, + StreamTableEnvironment tableEnv) + throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { //register urf - // udf和tableEnv须由同一个类加载器加载 - ClassLoader levelClassLoader = tableEnv.getClass().getClassLoader(); URLClassLoader classLoader = null; List funcList = sqlTree.getFunctionList(); for (CreateFuncParser.SqlParserResult funcInfo : funcList) { //classloader if (classLoader == null) { - classLoader = FlinkUtil.loadExtraJar(jarURList, (URLClassLoader)levelClassLoader); + classLoader = FlinkUtil.loadExtraJar(jarURList, parentClassloader); } - FlinkUtil.registerUDF(funcInfo.getType(), funcInfo.getClassName(), funcInfo.getName(), tableEnv, classLoader); + classLoader.loadClass(funcInfo.getClassName()); + FlinkUtil.registerUDF(funcInfo.getType(), funcInfo.getClassName(), funcInfo.getName(), + tableEnv, classLoader); } } - private static void registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, String localSqlPluginPath, - String remoteSqlPluginPath, String pluginLoadMode, Map sideTableMap, Map registerTableCache) throws Exception { + private static void registerTable(SqlTree sqlTree, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv, + String localSqlPluginPath, String remoteSqlPluginPath, + Map sideTableMap, Map registerTableCache) throws Exception { Set classPathSet = Sets.newHashSet(); WaterMarkerAssigner waterMarkerAssigner = new WaterMarkerAssigner(); for (TableInfo tableInfo : sqlTree.getTableInfoMap().values()) { @@ -249,10 +279,7 @@ private static void registerTable(SqlTree sqlTree, StreamExecutionEnvironment en Table adaptTable = adaptSql == null ? table : tableEnv.sqlQuery(adaptSql); RowTypeInfo typeInfo = new RowTypeInfo(adaptTable.getSchema().getTypes(), adaptTable.getSchema().getColumnNames()); - DataStream adaptStream = tableEnv.toRetractStream(adaptTable, typeInfo) - .map((Tuple2 f0) -> { return f0.f1; }) - .returns(typeInfo); - + DataStream adaptStream = tableEnv.toAppendStream(adaptTable, typeInfo); String fields = String.join(",", typeInfo.getFieldNames()); if(waterMarkerAssigner.checkNeedAssignWaterMarker(sourceTableInfo)){ @@ -264,22 +291,19 @@ private static void registerTable(SqlTree sqlTree, StreamExecutionEnvironment en Table regTable = tableEnv.fromDataStream(adaptStream, fields); tableEnv.registerTable(tableInfo.getName(), regTable); - if(LOG.isInfoEnabled()){ - LOG.info("registe table {} success.", tableInfo.getName()); - } registerTableCache.put(tableInfo.getName(), regTable); - classPathSet.add(buildSourceAndSinkPathByLoadMode(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode)); + classPathSet.add(PluginUtil.getRemoteJarFilePath(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, remoteSqlPluginPath)); } else if (tableInfo instanceof TargetTableInfo) { TableSink tableSink = StreamSinkFactory.getTableSink((TargetTableInfo) tableInfo, localSqlPluginPath); TypeInformation[] flinkTypes = FlinkUtil.transformTypes(tableInfo.getFieldClasses()); tableEnv.registerTableSink(tableInfo.getName(), tableInfo.getFields(), flinkTypes, tableSink); - classPathSet.add(buildSourceAndSinkPathByLoadMode(tableInfo.getType(), TargetTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode)); + classPathSet.add( PluginUtil.getRemoteJarFilePath(tableInfo.getType(), TargetTableInfo.TARGET_SUFFIX, remoteSqlPluginPath)); } else if(tableInfo instanceof SideTableInfo){ String sideOperator = ECacheType.ALL.name().equals(((SideTableInfo) tableInfo).getCacheType()) ? "all" : "async"; sideTableMap.put(tableInfo.getName(), (SideTableInfo) tableInfo); - classPathSet.add(buildSidePathByLoadMode(tableInfo.getType(), sideOperator, SideTableInfo.TARGET_SUFFIX, localSqlPluginPath, remoteSqlPluginPath, pluginLoadMode)); + classPathSet.add(PluginUtil.getRemoteSideJarFilePath(tableInfo.getType(), sideOperator, SideTableInfo.TARGET_SUFFIX, remoteSqlPluginPath)); }else { throw new RuntimeException("not support table type:" + tableInfo.getType()); } @@ -295,36 +319,25 @@ private static void registerTable(SqlTree sqlTree, StreamExecutionEnvironment en } } - private static URL buildSourceAndSinkPathByLoadMode(String type, String suffix, String localSqlPluginPath, String remoteSqlPluginPath, String pluginLoadMode) throws Exception { - if (StringUtils.equalsIgnoreCase(pluginLoadMode, EPluginLoadMode.CLASSPATH.name())) { - return PluginUtil.getRemoteJarFilePath(type, suffix, remoteSqlPluginPath, localSqlPluginPath); - } - return PluginUtil.getLocalJarFilePath(type, suffix, localSqlPluginPath); - } - - private static URL buildSidePathByLoadMode(String type, String operator, String suffix, String localSqlPluginPath, String remoteSqlPluginPath, String pluginLoadMode) throws Exception { - if (StringUtils.equalsIgnoreCase(pluginLoadMode, EPluginLoadMode.CLASSPATH.name())) { - return PluginUtil.getRemoteSideJarFilePath(type, operator, suffix, remoteSqlPluginPath, localSqlPluginPath); - } - return PluginUtil.getLocalSideJarFilePath(type, operator, suffix, localSqlPluginPath); - } - - private static StreamExecutionEnvironment getStreamExeEnv(Properties confProperties, String deployMode) throws Exception { - confProperties = PropertiesUtils.propertiesTrim(confProperties); - + private static StreamExecutionEnvironment getStreamExeEnv(Properties confProperties, String deployMode) throws IOException, NoSuchMethodException { StreamExecutionEnvironment env = !ClusterMode.local.name().equals(deployMode) ? StreamExecutionEnvironment.getExecutionEnvironment() : new MyLocalStreamEnvironment(); - env.getConfig().disableClosureCleaner(); - env.setParallelism(FlinkUtil.getEnvParallelism(confProperties)); + env.setParallelism(FlinkUtil.getEnvParallelism(confProperties)); Configuration globalJobParameters = new Configuration(); - //Configuration unsupported set properties key-value Method method = Configuration.class.getDeclaredMethod("setValueInternal", String.class, Object.class); method.setAccessible(true); - for (Map.Entry prop : confProperties.entrySet()) { - method.invoke(globalJobParameters, prop.getKey(), prop.getValue()); - } + + confProperties.forEach((key,val) -> { + try { + method.invoke(globalJobParameters, key, val); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } + }); ExecutionConfig exeConfig = env.getConfig(); if(exeConfig.getGlobalJobParameters() == null){ @@ -332,20 +345,25 @@ private static StreamExecutionEnvironment getStreamExeEnv(Properties confPropert }else if(exeConfig.getGlobalJobParameters() instanceof Configuration){ ((Configuration) exeConfig.getGlobalJobParameters()).addAll(globalJobParameters); } + + if(FlinkUtil.getMaxEnvParallelism(confProperties) > 0){ env.setMaxParallelism(FlinkUtil.getMaxEnvParallelism(confProperties)); } + if(FlinkUtil.getBufferTimeoutMillis(confProperties) > 0){ env.setBufferTimeout(FlinkUtil.getBufferTimeoutMillis(confProperties)); } + env.setRestartStrategy(RestartStrategies.failureRateRestart( - ConfigConstrant.failureRate, - Time.of(ConfigConstrant.failureInterval, TimeUnit.MINUTES), - Time.of(ConfigConstrant.delayInterval, TimeUnit.SECONDS) + failureRate, + Time.of(failureInterval, TimeUnit.MINUTES), + Time.of(delayInterval, TimeUnit.SECONDS) )); + FlinkUtil.setStreamTimeCharacteristic(env, confProperties); FlinkUtil.openCheckpoint(env, confProperties); + return env; } - } diff --git a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java b/core/src/main/java/com/dtstack/flink/sql/MyLocalStreamEnvironment.java similarity index 85% rename from core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java rename to core/src/main/java/com/dtstack/flink/sql/MyLocalStreamEnvironment.java index 8ac1edd41..54ddaa647 100644 --- a/core/src/main/java/com/dtstack/flink/sql/environment/MyLocalStreamEnvironment.java +++ b/core/src/main/java/com/dtstack/flink/sql/MyLocalStreamEnvironment.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.dtstack.flink.sql.environment; +package com.dtstack.flink.sql; import org.apache.flink.api.common.InvalidProgramException; import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.minicluster.MiniCluster; -import org.apache.flink.runtime.minicluster.MiniClusterConfiguration; +import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; import org.apache.flink.streaming.api.environment.LocalStreamEnvironment; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.graph.StreamGraph; @@ -100,27 +100,24 @@ public JobExecutionResult execute(String jobName) throws Exception { Configuration configuration = new Configuration(); configuration.addAll(jobGraph.getJobConfiguration()); - configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "512M"); - configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); + configuration.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, -1L); + configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); // add (and override) the settings with what the user defined configuration.addAll(this.conf); - MiniClusterConfiguration.Builder configBuilder = new MiniClusterConfiguration.Builder(); - configBuilder.setConfiguration(configuration); - if (LOG.isInfoEnabled()) { LOG.info("Running job on local embedded Flink mini cluster"); } - MiniCluster exec = new MiniCluster(configBuilder.build()); + LocalFlinkMiniCluster exec = new LocalFlinkMiniCluster(configuration, true); try { exec.start(); - return exec.executeJobBlocking(jobGraph); + return exec.submitJobAndWait(jobGraph, getConfig().isSysoutLoggingEnabled()); } finally { transformations.clear(); - exec.closeAsync(); + exec.stop(); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java deleted file mode 100644 index 6db1058e5..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderManager.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.classloader; - -import com.dtstack.flink.sql.util.PluginUtil; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * company: www.dtstack.com - * author: toutian - * create: 2019/10/14 - */ -public class ClassLoaderManager { - - private static final Logger LOG = LoggerFactory.getLogger(ClassLoaderManager.class); - - private static Map pluginClassLoader = new ConcurrentHashMap<>(); - - public static R newInstance(String pluginJarPath, ClassLoaderSupplier supplier) throws Exception { - ClassLoader classLoader = retrieveClassLoad(pluginJarPath); - return ClassLoaderSupplierCallBack.callbackAndReset(supplier, classLoader); - } - - public static R newInstance(List jarUrls, ClassLoaderSupplier supplier) throws Exception { - ClassLoader classLoader = retrieveClassLoad(jarUrls); - return ClassLoaderSupplierCallBack.callbackAndReset(supplier, classLoader); - } - - private static DtClassLoader retrieveClassLoad(String pluginJarPath) { - return pluginClassLoader.computeIfAbsent(pluginJarPath, k -> { - try { - URL[] urls = PluginUtil.getPluginJarUrls(pluginJarPath); - ClassLoader parentClassLoader = Thread.currentThread().getContextClassLoader(); - DtClassLoader classLoader = new DtClassLoader(urls, parentClassLoader); - LOG.info("pluginJarPath:{} create ClassLoad successful...", pluginJarPath); - return classLoader; - } catch (Throwable e) { - LOG.error("retrieve ClassLoad happens error:{}", e); - throw new RuntimeException("retrieve ClassLoad happens error"); - } - }); - } - - private static DtClassLoader retrieveClassLoad(List jarUrls) { - jarUrls.sort(Comparator.comparing(URL::toString)); - String jarUrlkey = StringUtils.join(jarUrls, "_"); - return pluginClassLoader.computeIfAbsent(jarUrlkey, k -> { - try { - URL[] urls = jarUrls.toArray(new URL[jarUrls.size()]); - ClassLoader parentClassLoader = Thread.currentThread().getContextClassLoader(); - DtClassLoader classLoader = new DtClassLoader(urls, parentClassLoader); - LOG.info("jarUrl:{} create ClassLoad successful...", jarUrlkey); - return classLoader; - } catch (Throwable e) { - LOG.error("retrieve ClassLoad happens error:{}", e); - throw new RuntimeException("retrieve ClassLoad happens error"); - } - }); - } - - public static List getClassPath() { - List classPaths = new ArrayList<>(); - for (Map.Entry entry : pluginClassLoader.entrySet()) { - classPaths.addAll(Arrays.asList(entry.getValue().getURLs())); - } - return classPaths; - } -} diff --git a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderSupplier.java b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderSupplier.java deleted file mode 100644 index 859aa75f4..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderSupplier.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.classloader; - -/** - * Represents a supplier of results. - * - *

There is no requirement that a new or distinct result be returned each - * time the supplier is invoked. - * - *

This is a functional interface - * whose functional method is {@link #get()}. - * - * @param the type of results supplied by this supplier - * - * @since 1.8 - */ -@FunctionalInterface -public interface ClassLoaderSupplier { - - /** - * Gets a result. - * - * @return a result - */ - T get(ClassLoader cl) throws Exception; -} diff --git a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderSupplierCallBack.java b/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderSupplierCallBack.java deleted file mode 100644 index 51d37ef5e..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/classloader/ClassLoaderSupplierCallBack.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.classloader; - -/** - * company: www.dtstack.com - * author: toutian - * create: 2019/10/14 - */ -public class ClassLoaderSupplierCallBack { - - public static R callbackAndReset(ClassLoaderSupplier supplier, ClassLoader toSetClassLoader) throws Exception { - ClassLoader oldClassLoader = Thread.currentThread().getContextClassLoader(); - Thread.currentThread().setContextClassLoader(toSetClassLoader); - try { - return supplier.get(toSetClassLoader); - } finally { - Thread.currentThread().setContextClassLoader(oldClassLoader); - } - } - - -} diff --git a/core/src/main/java/com/dtstack/flink/sql/config/CalciteConfig.java b/core/src/main/java/com/dtstack/flink/sql/config/CalciteConfig.java deleted file mode 100644 index 54ae66bbc..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/config/CalciteConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.config; - -import org.apache.calcite.config.Lex; -import org.apache.calcite.sql.parser.SqlParser; -import org.apache.calcite.sql.parser.SqlParser.Config; - -public class CalciteConfig { - - public static Config MYSQL_LEX_CONFIG = SqlParser - .configBuilder() - .setLex(Lex.MYSQL) - .build(); - - - -} diff --git a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java b/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java deleted file mode 100644 index 6cb027ac3..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/enums/EPluginLoadMode.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.enums; - -/** - * - * CLASSPATH： plugin jar depends on each machine node. - * SHIPFILE: plugin jar only depends on the client submitted by the task. - * - */ -public enum EPluginLoadMode { - - CLASSPATH(0), - SHIPFILE(1); - - private int type; - - EPluginLoadMode(int type){ - this.type = type; - } - - public int getType(){ - return this.type; - } -} diff --git a/core/src/main/java/com/dtstack/flink/sql/exec/FlinkSQLExec.java b/core/src/main/java/com/dtstack/flink/sql/exec/FlinkSQLExec.java deleted file mode 100644 index 6bcc25251..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/exec/FlinkSQLExec.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.exec; - -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlInsert; -import org.apache.calcite.sql.SqlNode; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.TableException; -import org.apache.flink.table.api.ValidationException; -import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.table.calcite.FlinkPlannerImpl; -import org.apache.flink.table.plan.logical.LogicalRelNode; -import org.apache.flink.table.plan.schema.TableSinkTable; -import org.apache.flink.table.plan.schema.TableSourceSinkTable; -import scala.Option; - -import java.lang.reflect.Method; - -/** - * @description: mapping by name when insert into sink table - * @author: maqi - * @create: 2019/08/15 11:09 - */ -public class FlinkSQLExec { - - public static void sqlUpdate(StreamTableEnvironment tableEnv, String stmt) throws Exception { - - FlinkPlannerImpl planner = new FlinkPlannerImpl(tableEnv.getFrameworkConfig(), tableEnv.getPlanner(), tableEnv.getTypeFactory()); - SqlNode insert = planner.parse(stmt); - - if (!(insert instanceof SqlInsert)) { - throw new TableException( - "Unsupported SQL query! sqlUpdate() only accepts SQL statements of type INSERT."); - } - SqlNode query = ((SqlInsert) insert).getSource(); - - SqlNode validatedQuery = planner.validate(query); - - Table queryResult = new Table(tableEnv, new LogicalRelNode(planner.rel(validatedQuery).rel)); - String targetTableName = ((SqlIdentifier) ((SqlInsert) insert).getTargetTable()).names.get(0); - - Method method = TableEnvironment.class.getDeclaredMethod("getTable", String.class); - method.setAccessible(true); - Option sinkTab = (Option)method.invoke(tableEnv, targetTableName); - - if (sinkTab.isEmpty()) { - throw new ValidationException("Sink table " + targetTableName + "not found in flink"); - } - - TableSourceSinkTable targetTable = (TableSourceSinkTable) sinkTab.get(); - TableSinkTable tableSinkTable = (TableSinkTable)targetTable.tableSinkTable().get(); - String[] fieldNames = tableSinkTable.tableSink().getFieldNames(); - - Table newTable = null; - try { - newTable = queryResult.select(String.join(",", fieldNames)); - } catch (Exception e) { - throw new ValidationException( - "Field name of query result and registered TableSink "+targetTableName +" do not match.\n" + - "Query result schema: " + String.join(",", queryResult.getSchema().getColumnNames()) + "\n" + - "TableSink schema: " + String.join(",", fieldNames)); - } - - tableEnv.insertInto(newTable, targetTableName, tableEnv.queryConfig()); - } -} \ No newline at end of file diff --git a/core/src/main/java/com/dtstack/flink/sql/metric/MetricConstant.java b/core/src/main/java/com/dtstack/flink/sql/metric/MetricConstant.java index 17bb75e82..89c411479 100644 --- a/core/src/main/java/com/dtstack/flink/sql/metric/MetricConstant.java +++ b/core/src/main/java/com/dtstack/flink/sql/metric/MetricConstant.java @@ -45,8 +45,6 @@ public class MetricConstant { public static final String DT_NUM_RECORDS_OUT = "dtNumRecordsOut"; - public static final String DT_NUM_DIRTY_RECORDS_OUT = "dtNumDirtyRecordsOut"; - public static final String DT_NUM_RECORDS_OUT_RATE = "dtNumRecordsOutRate"; public static final String DT_EVENT_DELAY_GAUGE = "dtEventDelay"; diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java deleted file mode 100644 index 07860b608..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionParser.java +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.option; - -import com.google.common.collect.Lists; -import com.dtstack.flink.sql.util.PluginUtil; -import org.apache.commons.cli.BasicParser; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.ParseException; -import org.apache.commons.lang.StringUtils; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Field; -import java.util.List; -import java.util.Map; -import java.io.File; -import java.io.FileInputStream; -import java.net.URLEncoder; -import org.apache.commons.codec.Charsets; - - -/** - * The Parser of Launcher commandline options - * - * Company: www.dtstack.com - * @author sishu.yss - */ -public class OptionParser { - - public static final String OPTION_SQL = "sql"; - - private org.apache.commons.cli.Options options = new org.apache.commons.cli.Options(); - - private BasicParser parser = new BasicParser(); - - private Options properties = new Options(); - - public OptionParser(String[] args) throws Exception { - initOptions(addOptions(args)); - } - - private CommandLine addOptions(String[] args) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, ParseException { - Class cla = properties.getClass(); - Field[] fields = cla.getDeclaredFields(); - for(Field field:fields){ - String name = field.getName(); - OptionRequired optionRequired = field.getAnnotation(OptionRequired.class); - if(optionRequired != null){ - options.addOption(name,optionRequired.hasArg(),optionRequired.description()); - } - } - CommandLine cl = parser.parse(options, args); - return cl; - } - - private void initOptions(CommandLine cl) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, ParseException { - Class cla = properties.getClass(); - Field[] fields = cla.getDeclaredFields(); - for(Field field:fields){ - String name = field.getName(); - String value = cl.getOptionValue(name); - OptionRequired optionRequired = field.getAnnotation(OptionRequired.class); - if(optionRequired != null){ - if(optionRequired.required()&&StringUtils.isBlank(value)){ - throw new RuntimeException(String.format("parameters of %s is required",name)); - } - } - if(StringUtils.isNotBlank(value)){ - field.setAccessible(true); - field.set(properties,value); - } - } - } - - public Options getOptions(){ - return properties; - } - - public List getProgramExeArgList() throws Exception { - Map mapConf = PluginUtil.ObjectToMap(properties); - List args = Lists.newArrayList(); - for(Map.Entry one : mapConf.entrySet()){ - String key = one.getKey(); - Object value = one.getValue(); - if(value == null){ - continue; - }else if(OPTION_SQL.equalsIgnoreCase(key)){ - File file = new File(value.toString()); - FileInputStream in = new FileInputStream(file); - byte[] filecontent = new byte[(int) file.length()]; - in.read(filecontent); - String content = new String(filecontent, Charsets.UTF_8.name()); - value = URLEncoder.encode(content, Charsets.UTF_8.name()); - } - args.add("-" + key); - args.add(value.toString()); - } - return args; - } - - public static void main(String[] args) throws Exception { - OptionParser OptionParser = new OptionParser(args); - System.out.println(OptionParser.getOptions()); - } -} diff --git a/core/src/main/java/com/dtstack/flink/sql/option/OptionRequired.java b/core/src/main/java/com/dtstack/flink/sql/option/OptionRequired.java deleted file mode 100644 index b70c4424b..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/option/OptionRequired.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.flink.sql.option; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * - * Reason: TODO ADD REASON(可选) - * Date: 2019年9月16日 下午1:24:39 - * Company: www.dtstack.com - * @author sishu.yss - * - */ -@Target({ElementType.FIELD}) -@Retention(RetentionPolicy.RUNTIME) -public @interface OptionRequired { - - boolean required() default false; - - boolean hasArg() default true; - - String description() default ""; -} diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java index 670d98a7e..9b7017743 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateFuncParser.java @@ -24,7 +24,7 @@ import java.util.regex.Pattern; /** - * parser register udf sql + * 解析创建自定义方法sql * Date: 2018/6/26 * Company: www.dtstack.com * @author xuchao @@ -32,7 +32,7 @@ public class CreateFuncParser implements IParser { - private static final String funcPatternStr = "(?i)\\s*create\\s+(scala|table|aggregate)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; + private static final String funcPatternStr = "(?i)\\s*create\\s+(scala|table)\\s+function\\s+(\\S+)\\s+WITH\\s+(\\S+)"; private static final Pattern funcPattern = Pattern.compile(funcPatternStr); diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java index ae6e1f708..b5e4a4aa4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTableParser.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.parser; import com.dtstack.flink.sql.util.DtStringUtil; -import com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.collect.Maps; import java.util.List; import java.util.Map; @@ -29,7 +29,7 @@ import java.util.regex.Pattern; /** - * parser create table sql + * 解析创建表结构sql * Date: 2018/6/26 * Company: www.dtstack.com * @author xuchao diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java index de7141eb5..cb5620d0a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java @@ -25,19 +25,14 @@ import org.apache.calcite.sql.*; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; -import com.google.common.collect.Lists; +import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; + import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.apache.calcite.sql.SqlKind.IDENTIFIER; -/** - * parser create tmp table sql - * Date: 2018/6/26 - * Company: www.dtstack.com - * @author yanxi - */ public class CreateTmpTableParser implements IParser { //select table tableName as select @@ -71,13 +66,11 @@ public void parseSql(String sql, SqlTree sqlTree) { tableName = matcher.group(1); selectSql = "select " + matcher.group(2); } - SqlParser.Config config = SqlParser .configBuilder() .setLex(Lex.MYSQL) .build(); SqlParser sqlParser = SqlParser.create(selectSql,config); - SqlNode sqlNode = null; try { sqlNode = sqlParser.parseStmt(); diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java index a7c6db9eb..52541385f 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/InsertSqlParser.java @@ -21,11 +21,16 @@ package com.dtstack.flink.sql.parser; import org.apache.calcite.config.Lex; -import org.apache.calcite.sql.*; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlInsert; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import org.apache.commons.lang3.StringUtils; -import com.google.common.collect.Lists; +import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; import java.util.List; @@ -113,10 +118,6 @@ private static void parseNode(SqlNode sqlNode, SqlParseResult sqlParseResult){ sqlParseResult.addSourceTable(identifierNode.toString()); } break; - case MATCH_RECOGNIZE: - SqlMatchRecognize node = (SqlMatchRecognize) sqlNode; - sqlParseResult.addSourceTable(node.getTableRef().toString()); - break; case UNION: SqlNode unionLeft = ((SqlBasicCall)sqlNode).getOperands()[0]; SqlNode unionRight = ((SqlBasicCall)sqlNode).getOperands()[1]; @@ -131,10 +132,6 @@ private static void parseNode(SqlNode sqlNode, SqlParseResult sqlParseResult){ parseNode(unionRight, sqlParseResult); } break; - case ORDER_BY: - SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode; - parseNode(sqlOrderBy.query, sqlParseResult); - break; default: //do nothing break; diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java index a76c1b31a..7ab616632 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java @@ -25,8 +25,8 @@ import com.dtstack.flink.sql.table.TableInfoParser; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; -import com.google.common.collect.Lists; -import com.google.common.base.Strings; +import org.apache.flink.shaded.curator.org.apache.curator.shaded.com.google.common.collect.Lists; +import org.apache.flink.shaded.guava18.com.google.common.base.Strings; import java.util.List; import java.util.Set; @@ -88,7 +88,6 @@ public static SqlTree parseSql(String sql) throws Exception { sqlParser.parseSql(childSql, sqlTree); result = true; - break; } if(!result){ @@ -139,15 +138,12 @@ public static SqlTree parseSql(String sql) throws Exception { if (!sqlTree.getTableInfoMap().keySet().contains(tableName)){ CreateTableParser.SqlParserResult createTableResult = sqlTree.getPreDealTableMap().get(tableName); if(createTableResult == null){ - CreateTmpTableParser.SqlParserResult tmpTableResult = sqlTree.getTmpTableMap().get(tableName); - if (tmpTableResult == null){ - throw new RuntimeException("can't find table " + tableName); - } - } else { - TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), - createTableResult, LOCAL_SQL_PLUGIN_ROOT); - sqlTree.addTableInfo(tableName, tableInfo); + throw new RuntimeException("can't find table " + tableName); } + + TableInfo tableInfo = tableInfoParser.parseWithTableType(ETableType.SOURCE.getType(), + createTableResult, LOCAL_SQL_PLUGIN_ROOT); + sqlTree.addTableInfo(tableName, tableInfo); } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java index 1b64b7c68..3ed37c51e 100644 --- a/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java +++ b/core/src/main/java/com/dtstack/flink/sql/parser/SqlTree.java @@ -22,14 +22,14 @@ import com.dtstack.flink.sql.table.TableInfo; -import com.google.common.collect.Maps; -import com.google.common.collect.Lists; +import org.apache.flink.shaded.curator.org.apache.curator.shaded.com.google.common.collect.Maps; +import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; import java.util.List; import java.util.Map; /** - * parser sql to get the Sql Tree structure + * 解析sql获得的对象结构 * Date: 2018/6/25 * Company: www.dtstack.com * @author xuchao diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java index e788cf139..a185da1bd 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AllReqRow.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.factory.DTThreadFactory; +import com.dtstack.flink.sql.threadFactory.DTThreadFactory; import org.apache.flink.api.common.functions.RichFlatMapFunction; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java index a496730e4..6df1af1d5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java @@ -16,7 +16,7 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.side; @@ -28,12 +28,9 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.functions.async.RichAsyncFunction; -import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; import org.apache.flink.types.Row; -import java.util.Collection; import java.util.Collections; -import java.util.concurrent.TimeoutException; /** * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow @@ -53,18 +50,6 @@ public AsyncReqRow(SideInfo sideInfo){ this.sideInfo = sideInfo; } - @Override - public void timeout(Row input, ResultFuture resultFuture) throws Exception { - StreamRecordQueueEntry future = (StreamRecordQueueEntry)resultFuture; - try { - if (null == future.get()) { - new TimeoutException("Async function call has timed out."); - } - } catch (Exception e) { - throw new Exception(e); - } - } - private void initCache(){ SideTableInfo sideTableInfo = sideInfo.getSideTableInfo(); if(sideTableInfo.getCacheType() == null || ECacheType.NONE.name().equalsIgnoreCase(sideTableInfo.getCacheType())){ diff --git a/core/src/main/java/com/dtstack/flink/sql/side/FieldReplaceInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/FieldReplaceInfo.java index 37b23d046..bc716ddaa 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/FieldReplaceInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/FieldReplaceInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side; -import com.google.common.collect.HashBasedTable; +import org.apache.flink.calcite.shaded.com.google.common.collect.HashBasedTable; /** * Reason: diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java index 6fde02493..03dbde5a6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinInfo.java @@ -22,10 +22,9 @@ import org.apache.calcite.sql.JoinType; import org.apache.calcite.sql.SqlNode; -import com.google.common.base.Strings; +import org.apache.flink.calcite.shaded.com.google.common.base.Strings; import java.io.Serializable; -import java.util.Map; /** * Join信息 @@ -41,8 +40,6 @@ public class JoinInfo implements Serializable { //左表是否是维表 private boolean leftIsSideTable; - //左表是 转换后的中间表 - private boolean leftIsMidTable; //右表是否是维表 private boolean rightIsSideTable; @@ -66,8 +63,6 @@ public class JoinInfo implements Serializable { private SqlNode selectNode; private JoinType joinType; - // 左边是中间转换表，做表映射关系，给替换属性名称使用 - private Map leftTabMapping; public String getSideTableName(){ if(leftIsSideTable){ @@ -92,22 +87,6 @@ public String getNewTableName(){ return leftStr + "_" + rightTableName; } - public boolean isLeftIsMidTable() { - return leftIsMidTable; - } - - public void setLeftIsMidTable(boolean leftIsMidTable) { - this.leftIsMidTable = leftIsMidTable; - } - - public Map getLeftTabMapping() { - return leftTabMapping; - } - - public void setLeftTabMapping(Map leftTabMapping) { - this.leftTabMapping = leftTabMapping; - } - public String getNewTableAlias(){ return leftTableAlias + "_" + rightTableAlias; } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinScope.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinScope.java index c7a73e0d7..ba07e714a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/JoinScope.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/JoinScope.java @@ -23,8 +23,8 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Maps; import java.util.List; import java.util.Map; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java index 74d303c24..df242a390 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/ParserJoinField.java @@ -27,7 +27,7 @@ import org.apache.calcite.sql.SqlSelect; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import java.util.Iterator; import java.util.List; @@ -41,12 +41,8 @@ public class ParserJoinField { - /** - * build row by field - * @param sqlNode select node - * @param scope join left and right table all info - * @param getAll true,get all fields from two tables; false, extract useful field from select node + * Need to parse the fields of information and where selectlist * @return */ public static List getRowTypeInfo(SqlNode sqlNode, JoinScope scope, boolean getAll){ diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java index df41e1663..924ef53f2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java @@ -27,8 +27,8 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Maps; import java.io.Serializable; import java.util.List; @@ -89,7 +89,7 @@ public void parseSelectFields(JoinInfo joinInfo){ for( int i=0; i getExeQueue(String exeSql, Set sideTableSet) throws SqlParseException { System.out.println("---exeSql---"); System.out.println(exeSql); - LOG.info("---exeSql---"); - LOG.info(exeSql); - Queue queueInfo = Queues.newLinkedBlockingQueue(); - SqlParser sqlParser = SqlParser.create(exeSql, CalciteConfig.MYSQL_LEX_CONFIG); + SqlParser.Config config = SqlParser + .configBuilder() + .setLex(Lex.MYSQL) + .build(); + SqlParser sqlParser = SqlParser.create(exeSql,config); SqlNode sqlNode = sqlParser.parseStmt(); parseSql(sqlNode, sideTableSet, queueInfo); queueInfo.offer(sqlNode); @@ -86,17 +70,6 @@ public Queue getExeQueue(String exeSql, Set sideTableSet) throws private Object parseSql(SqlNode sqlNode, Set sideTableSet, Queue queueInfo){ SqlKind sqlKind = sqlNode.getKind(); switch (sqlKind){ - case WITH: { - SqlWith sqlWith = (SqlWith) sqlNode; - SqlNodeList sqlNodeList = sqlWith.withList; - for (SqlNode withAsTable : sqlNodeList) { - SqlWithItem sqlWithItem = (SqlWithItem) withAsTable; - parseSql(sqlWithItem.query, sideTableSet, queueInfo); - queueInfo.add(sqlWithItem); - } - parseSql(sqlWith.body, sideTableSet, queueInfo); - break; - } case INSERT: SqlNode sqlSource = ((SqlInsert)sqlNode).getSource(); return parseSql(sqlSource, sideTableSet, queueInfo); @@ -137,66 +110,23 @@ private Object parseSql(SqlNode sqlNode, Set sideTableSet, Queue aliasInfo.setAlias(alias.toString()); return aliasInfo; - - case UNION: - SqlNode unionLeft = ((SqlBasicCall)sqlNode).getOperands()[0]; - SqlNode unionRight = ((SqlBasicCall)sqlNode).getOperands()[1]; - - parseSql(unionLeft, sideTableSet, queueInfo); - - parseSql(unionRight, sideTableSet, queueInfo); - - break; - - case ORDER_BY: - SqlOrderBy sqlOrderBy = (SqlOrderBy) sqlNode; - parseSql(sqlOrderBy.query, sideTableSet, queueInfo); } return ""; } - private JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, Queue queueInfo) { + private JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, Queue queueInfo){ SqlNode leftNode = joinNode.getLeft(); SqlNode rightNode = joinNode.getRight(); JoinType joinType = joinNode.getJoinType(); String leftTbName = ""; String leftTbAlias = ""; - String rightTableName = ""; - String rightTableAlias = ""; - Map midTableMapping = null ; - boolean leftIsMidTable = false; - // 右节点已经被解析 - boolean rightIsParse = false; - Tuple2 rightTableNameAndAlias = null; - if(leftNode.getKind() == IDENTIFIER){ leftTbName = leftNode.toString(); }else if(leftNode.getKind() == JOIN){ - JoinInfo leftNodeJoinInfo = (JoinInfo)parseSql(leftNode, sideTableSet, queueInfo);//解析多JOIN - - rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo); - rightIsParse = true; - if (checkIsSideTable(rightTableNameAndAlias.f0, sideTableSet)) { - // select * from xxx - SqlNode sqlNode = buildSelectByLeftNode(leftNode); - // ( select * from xxx) as xxx_0 - SqlBasicCall newAsNode = buildAsNodeByJoinInfo(leftNodeJoinInfo, sqlNode); - leftNode = newAsNode; - joinNode.setLeft(leftNode); - - leftIsMidTable = true; - midTableMapping = saveTabMapping(leftNodeJoinInfo); - - AliasInfo aliasInfo = (AliasInfo) parseSql(newAsNode, sideTableSet, queueInfo); - leftTbName = aliasInfo.getName(); - leftTbAlias = aliasInfo.getAlias(); - } else { - leftTbName = leftNodeJoinInfo.getRightTableName(); - leftTbAlias = leftNodeJoinInfo.getRightTableAlias(); - } - + Object leftNodeJoinInfo = parseSql(leftNode, sideTableSet, queueInfo); + System.out.println(leftNodeJoinInfo); }else if(leftNode.getKind() == AS){ AliasInfo aliasInfo = (AliasInfo) parseSql(leftNode, sideTableSet, queueInfo); leftTbName = aliasInfo.getName(); @@ -210,36 +140,27 @@ private JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, Queue< throw new RuntimeException("side-table must be at the right of join operator"); } - if (!rightIsParse) { - rightTableNameAndAlias = parseRightNode(rightNode, sideTableSet, queueInfo); + String rightTableName = ""; + String rightTableAlias = ""; + + if(rightNode.getKind() == IDENTIFIER){ + rightTableName = rightNode.toString(); + }else{ + AliasInfo aliasInfo = (AliasInfo)parseSql(rightNode, sideTableSet, queueInfo); + rightTableName = aliasInfo.getName(); + rightTableAlias = aliasInfo.getAlias(); } - rightTableName = rightTableNameAndAlias.f0; - rightTableAlias = rightTableNameAndAlias.f1; boolean rightIsSide = checkIsSideTable(rightTableName, sideTableSet); if(joinType == JoinType.RIGHT){ throw new RuntimeException("side join not support join type of right[current support inner join and left join]"); } - if (leftIsMidTable) { - // 替换右边 on语句 中的字段别名 - SqlNode afterReplaceNameCondition = ParseUtils.replaceJoinConditionTabName(joinNode.getCondition(), midTableMapping); - joinNode.setOperand(5, afterReplaceNameCondition); - } - JoinInfo tableInfo = new JoinInfo(); tableInfo.setLeftTableName(leftTbName); tableInfo.setRightTableName(rightTableName); - if (leftTbAlias.equals("")){ - tableInfo.setLeftTableAlias(leftTbName); - } else { - tableInfo.setLeftTableAlias(leftTbAlias); - } - if (leftTbAlias.equals("")){ - tableInfo.setRightTableAlias(rightTableName); - } else { - tableInfo.setRightTableAlias(rightTableAlias); - } + tableInfo.setLeftTableAlias(leftTbAlias); + tableInfo.setRightTableAlias(rightTableAlias); tableInfo.setLeftIsSideTable(leftIsSide); tableInfo.setRightIsSideTable(rightIsSide); tableInfo.setLeftNode(leftNode); @@ -247,105 +168,11 @@ private JoinInfo dealJoinNode(SqlJoin joinNode, Set sideTableSet, Queue< tableInfo.setJoinType(joinType); tableInfo.setCondition(joinNode.getCondition()); - tableInfo.setLeftIsMidTable(leftIsMidTable); - tableInfo.setLeftTabMapping(midTableMapping); - return tableInfo; } - private Tuple2 parseRightNode(SqlNode sqlNode, Set sideTableSet, Queue queueInfo) { - Tuple2 tabName = new Tuple2<>("", ""); - if(sqlNode.getKind() == IDENTIFIER){ - tabName.f0 = sqlNode.toString(); - }else{ - AliasInfo aliasInfo = (AliasInfo)parseSql(sqlNode, sideTableSet, queueInfo); - tabName.f0 = aliasInfo.getName(); - tabName.f1 = aliasInfo.getAlias(); - } - return tabName; - } - - private Map saveTabMapping(JoinInfo leftNodeJoinInfo) { - Map midTableMapping = Maps.newHashMap();; - - String midTab = buidTableName(leftNodeJoinInfo.getLeftTableAlias(), SPLIT, leftNodeJoinInfo.getRightTableAlias()); - String finalMidTab = midTab + "_0"; - - if(leftNodeJoinInfo.isLeftIsMidTable()) { - midTableMapping.putAll(leftNodeJoinInfo.getLeftTabMapping()); - } - fillLeftAllTable(leftNodeJoinInfo, midTableMapping, finalMidTab); - return midTableMapping; - } - - private void fillLeftAllTable(JoinInfo leftNodeJoinInfo, Map midTableMapping, String finalMidTab) { - List tablesName = Lists.newArrayList(); - ParseUtils.parseLeftNodeTableName(leftNodeJoinInfo.getLeftNode(), tablesName); - - tablesName.forEach(tab ->{ - midTableMapping.put(tab, finalMidTab); - }); - midTableMapping.put(leftNodeJoinInfo.getRightTableAlias(), finalMidTab); - } - - - private SqlNode buildSelectByLeftNode(SqlNode leftNode) { - SqlParser sqlParser = SqlParser.create(tempSQL, CalciteConfig.MYSQL_LEX_CONFIG); - SqlNode sqlNode = null; - try { - sqlNode = sqlParser.parseStmt(); - }catch (Exception e) { - LOG.error("tmp sql parse error..", e); - } - - ((SqlSelect) sqlNode).setFrom(leftNode); - return sqlNode; - } - private void dealSelectResultWithJoinInfo(JoinInfo joinInfo, SqlSelect sqlNode, Queue queueInfo){ - // 中间虚拟表进行表名称替换 - if (joinInfo.isLeftIsMidTable()){ - SqlNode whereNode = sqlNode.getWhere(); - SqlNodeList sqlGroup = sqlNode.getGroup(); - SqlNodeList sqlSelectList = sqlNode.getSelectList(); - List newSelectNodeList = Lists.newArrayList(); - - for( int i=0; i sideTableList){ if(sideTableList.contains(tableName)){ return true; } + return false; } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java index bfccd02f7..3623ecfd4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideSqlExec.java @@ -16,18 +16,15 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.side; import com.dtstack.flink.sql.enums.ECacheType; -import com.dtstack.flink.sql.exec.FlinkSQLExec; import com.dtstack.flink.sql.parser.CreateTmpTableParser; import com.dtstack.flink.sql.side.operator.SideAsyncOperator; import com.dtstack.flink.sql.side.operator.SideWithAllCacheOperator; import com.dtstack.flink.sql.util.ClassUtil; -import com.dtstack.flink.sql.util.ParseUtils; -import org.apache.calcite.sql.SqlAsOperator; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlIdentifier; @@ -37,26 +34,19 @@ import org.apache.calcite.sql.SqlLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSelect; -import org.apache.calcite.sql.SqlWithItem; import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.commons.collections.CollectionUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.collect.HashBasedTable; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Maps; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.types.Row; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.*; @@ -71,8 +61,6 @@ public class SideSqlExec { - private static final Logger LOG = LoggerFactory.getLogger(SideSqlExec.class); - private String localSqlPluginPath = null; private String tmpFields = null; @@ -104,31 +92,18 @@ public void exec(String sql, Map sideTableMap, StreamTabl if(preIsSideJoin){ preIsSideJoin = false; - List fieldNames = null; for(FieldReplaceInfo replaceInfo : replaceInfoList){ - fieldNames = Lists.newArrayList(); replaceFieldName(pollSqlNode, replaceInfo.getMappingTable(), replaceInfo.getTargetTableName(), replaceInfo.getTargetTableAlias()); - addAliasForFieldNode(pollSqlNode, fieldNames, replaceInfo.getMappingTable()); } } if(pollSqlNode.getKind() == INSERT){ - System.out.println("----------real exec sql-----------" ); - System.out.println(pollSqlNode.toString()); - FlinkSQLExec.sqlUpdate(tableEnv, pollSqlNode.toString()); - if(LOG.isInfoEnabled()){ - LOG.info("exec sql: " + pollSqlNode.toString()); - } + tableEnv.sqlUpdate(pollSqlNode.toString()); }else if(pollSqlNode.getKind() == AS){ AliasInfo aliasInfo = parseASNode(pollSqlNode); - Table table = tableEnv.sqlQuery(aliasInfo.getName()); + Table table = tableEnv.sql(aliasInfo.getName()); tableEnv.registerTable(aliasInfo.getAlias(), table); localTableCache.put(aliasInfo.getAlias(), table); - } else if (pollSqlNode.getKind() == WITH_ITEM) { - SqlWithItem sqlWithItem = (SqlWithItem) pollSqlNode; - String TableAlias = sqlWithItem.name.toString(); - Table table = tableEnv.sqlQuery(sqlWithItem.query.toString()); - tableEnv.registerTable(TableAlias, table); } }else if (pollObj instanceof JoinInfo){ @@ -139,68 +114,6 @@ public void exec(String sql, Map sideTableMap, StreamTabl } - - private void addAliasForFieldNode(SqlNode pollSqlNode, List fieldList, HashBasedTable mappingTable) { - SqlKind sqlKind = pollSqlNode.getKind(); - switch (sqlKind) { - case INSERT: - SqlNode source = ((SqlInsert) pollSqlNode).getSource(); - addAliasForFieldNode(source, fieldList, mappingTable); - break; - - case AS: - addAliasForFieldNode(((SqlBasicCall) pollSqlNode).getOperands()[0], fieldList, mappingTable); - break; - - case SELECT: - - SqlNodeList selectList = ((SqlSelect) pollSqlNode).getSelectList(); - - selectList.getList().forEach(node -> { - if (node.getKind() == IDENTIFIER) { - SqlIdentifier sqlIdentifier = (SqlIdentifier) node; - if (sqlIdentifier.names.size() == 1) { - return; - } - // save real field - String fieldName = sqlIdentifier.names.get(1); - if (!fieldName.endsWith("0") || fieldName.endsWith("0") && mappingTable.columnMap().containsKey(fieldName)) { - fieldList.add(fieldName); - } - - } - }); - - for (int i = 0; i < selectList.getList().size(); i++) { - SqlNode node = selectList.get(i); - if (node.getKind() == IDENTIFIER) { - SqlIdentifier sqlIdentifier = (SqlIdentifier) node; - if (sqlIdentifier.names.size() == 1) { - return; - } - - String name = sqlIdentifier.names.get(1); - // avoid real field pv0 convert pv - if (name.endsWith("0") && !fieldList.contains(name) && !fieldList.contains(name.substring(0, name.length() - 1))) { - SqlOperator operator = new SqlAsOperator(); - SqlParserPos sqlParserPos = new SqlParserPos(0, 0); - - SqlIdentifier sqlIdentifierAlias = new SqlIdentifier(name.substring(0, name.length() - 1), null, sqlParserPos); - SqlNode[] sqlNodes = new SqlNode[2]; - sqlNodes[0] = sqlIdentifier; - sqlNodes[1] = sqlIdentifierAlias; - SqlBasicCall sqlBasicCall = new SqlBasicCall(operator, sqlNodes, sqlParserPos); - - selectList.set(i, sqlBasicCall); - } - - } - } - break; - } - } - - public AliasInfo parseASNode(SqlNode sqlNode) throws SqlParseException { SqlKind sqlKind = sqlNode.getKind(); if(sqlKind != AS){ @@ -247,7 +160,7 @@ private void replaceFieldName(SqlNode sqlNode, HashBasedTable mappingTable, String tableAlias){ if(groupNode.getKind() == IDENTIFIER){ SqlIdentifier sqlIdentifier = (SqlIdentifier) groupNode; - if(sqlIdentifier.names.size() == 1){ - return sqlIdentifier; - } String mappingFieldName = mappingTable.get(sqlIdentifier.getComponent(0).getSimple(), sqlIdentifier.getComponent(1).getSimple()); - - if(mappingFieldName == null){ - throw new RuntimeException("can't find mapping fieldName:" + sqlIdentifier.toString() ); - } sqlIdentifier = sqlIdentifier.setName(0, tableAlias); return sqlIdentifier.setName(1, mappingFieldName); }else if(groupNode instanceof SqlBasicCall){ @@ -387,7 +255,7 @@ private SqlNode replaceNodeInfo(SqlNode groupNode, HashBasedTable replaceSelectStarFieldName(SqlNode selectNode, HashBasedTa } } - private SqlNode replaceSelectFieldName(SqlNode selectNode, HashBasedTable mappingTable, String tableAlias) { - if (selectNode.getKind() == AS) { - SqlNode leftNode = ((SqlBasicCall) selectNode).getOperands()[0]; + private SqlNode replaceSelectFieldName(SqlNode selectNode, HashBasedTable mappingTable, String tableAlias){ + if(selectNode.getKind() == AS){ + SqlNode leftNode = ((SqlBasicCall)selectNode).getOperands()[0]; SqlNode replaceNode = replaceSelectFieldName(leftNode, mappingTable, tableAlias); - if (replaceNode != null) { - ((SqlBasicCall) selectNode).getOperands()[0] = replaceNode; + if(replaceNode != null){ + ((SqlBasicCall)selectNode).getOperands()[0] = replaceNode; } return selectNode; @@ -481,7 +349,7 @@ private SqlNode replaceSelectFieldName(SqlNode selectNode, HashBasedTable conditionFields = getConditionFields(conditionNode, sideTableAlias, sideTableInfo); - if(CollectionUtils.isEqualCollection(conditionFields, convertPrimaryAlias(sideTableInfo))){ + private boolean checkJoinCondition(SqlNode conditionNode, String sideTableAlias, List primaryKeys){ + + List conditionFields = getConditionFields(conditionNode, sideTableAlias); + if(CollectionUtils.isEqualCollection(conditionFields, primaryKeys)){ return true; } - return false; - } - private List convertPrimaryAlias(SideTableInfo sideTableInfo) { - List res = Lists.newArrayList(); - sideTableInfo.getPrimaryKeys().forEach(field -> { - res.add(sideTableInfo.getPhysicalFields().getOrDefault(field, field)); - }); - return res; + return false; } - public List getConditionFields(SqlNode conditionNode, String specifyTableName, SideTableInfo sideTableInfo){ + public List getConditionFields(SqlNode conditionNode, String specifyTableName){ List sqlNodeList = Lists.newArrayList(); - ParseUtils.parseAnd(conditionNode, sqlNodeList); + if(conditionNode.getKind() == SqlKind.AND){ + sqlNodeList.addAll(Lists.newArrayList(((SqlBasicCall)conditionNode).getOperands())); + }else{ + sqlNodeList.add(conditionNode); + } + List conditionFields = Lists.newArrayList(); for(SqlNode sqlNode : sqlNodeList){ - if (!SqlKind.COMPARISON.contains(sqlNode.getKind())) { - throw new RuntimeException("not compare operator."); + if(sqlNode.getKind() != SqlKind.EQUALS){ + throw new RuntimeException("not equal operator."); } SqlIdentifier left = (SqlIdentifier)((SqlBasicCall)sqlNode).getOperands()[0]; @@ -621,7 +487,7 @@ public List getConditionFields(SqlNode conditionNode, String specifyTabl }else{ throw new RuntimeException(String.format("side table:%s join condition is wrong", specifyTableName)); } - tableCol = sideTableInfo.getPhysicalFields().getOrDefault(tableCol, tableCol); + conditionFields.add(tableCol); } @@ -652,11 +518,8 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, if(preIsSideJoin){ preIsSideJoin = false; - List fieldNames = null; - for (FieldReplaceInfo replaceInfo : replaceInfoList) { - fieldNames = Lists.newArrayList(); + for(FieldReplaceInfo replaceInfo : replaceInfoList){ replaceFieldName(pollSqlNode, replaceInfo.getMappingTable(), replaceInfo.getTargetTableName(), replaceInfo.getTargetTableAlias()); - addAliasForFieldNode(pollSqlNode, fieldNames, replaceInfo.getMappingTable()); } } @@ -664,11 +527,8 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, tableEnv.sqlUpdate(pollSqlNode.toString()); }else if(pollSqlNode.getKind() == AS){ AliasInfo aliasInfo = parseASNode(pollSqlNode); - Table table = tableEnv.sqlQuery(aliasInfo.getName()); + Table table = tableEnv.sql(aliasInfo.getName()); tableEnv.registerTable(aliasInfo.getAlias(), table); - if(LOG.isInfoEnabled()){ - LOG.info("Register Table {} by {}", aliasInfo.getAlias(), aliasInfo.getName()); - } localTableCache.put(aliasInfo.getAlias(), table); } else if (pollSqlNode.getKind() == SELECT){ Table table = tableEnv.sqlQuery(pollObj.toString()); @@ -692,7 +552,6 @@ public void registerTmpTable(CreateTmpTableParser.SqlParserResult result, } } } - private void joinFun(Object pollObj, Map localTableCache, Map sideTableMap, StreamTableEnvironment tableEnv, List replaceInfoList) throws Exception{ @@ -719,7 +578,7 @@ private void joinFun(Object pollObj, Map localTableCache, throw new RuntimeException("can't not find side table:" + joinInfo.getRightTableName()); } - if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo)){ + if(!checkJoinCondition(joinInfo.getCondition(), joinInfo.getRightTableAlias(), sideTableInfo.getPrimaryKeys())){ throw new RuntimeException("ON condition must contain all equal fields!!!"); } @@ -738,14 +597,11 @@ private void joinFun(Object pollObj, Map localTableCache, } RowTypeInfo typeInfo = new RowTypeInfo(targetTable.getSchema().getTypes(), targetTable.getSchema().getColumnNames()); - - DataStream adaptStream = tableEnv.toRetractStream(targetTable, org.apache.flink.types.Row.class) - .map((Tuple2 f0) -> { return f0.f1; }) - .returns(Row.class); + DataStream adaptStream = tableEnv.toAppendStream(targetTable, org.apache.flink.types.Row.class); //join side table before keyby ===> Reducing the size of each dimension table cache of async if(sideTableInfo.isPartitionedJoin()){ - List leftJoinColList = getConditionFields(joinInfo.getCondition(), joinInfo.getLeftTableAlias(), sideTableInfo); + List leftJoinColList = getConditionFields(joinInfo.getCondition(), joinInfo.getLeftTableAlias()); String[] leftJoinColArr = new String[leftJoinColList.size()]; leftJoinColArr = leftJoinColList.toArray(leftJoinColArr); adaptStream = adaptStream.keyBy(leftJoinColArr); @@ -776,11 +632,12 @@ private void joinFun(Object pollObj, Map localTableCache, } } - private boolean checkFieldsInfo(CreateTmpTableParser.SqlParserResult result, Table table) { + private boolean checkFieldsInfo(CreateTmpTableParser.SqlParserResult result, Table table){ List fieldNames = new LinkedList<>(); String fieldsInfo = result.getFieldsInfoStr(); String[] fields = fieldsInfo.split(","); - for (int i = 0; i < fields.length; i++) { + for (int i=0; i < fields.length; i++) + { String[] filed = fields[i].split("\\s"); if (filed.length < 2 || fields.length != table.getSchema().getColumnNames().length){ return false; @@ -791,7 +648,7 @@ private boolean checkFieldsInfo(CreateTmpTableParser.SqlParserResult result, Tab fieldNames.add(fieldName); String fieldType = filed[filed.length - 1 ].trim(); Class fieldClass = ClassUtil.stringConvertClass(fieldType); - Class tableField = table.getSchema().getFieldType(i).get().getTypeClass(); + Class tableField = table.getSchema().getType(i).get().getTypeClass(); if (fieldClass == tableField){ continue; } else { diff --git a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java index e21389ea7..0abd55a92 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/SideTableInfo.java @@ -45,26 +45,14 @@ public abstract class SideTableInfo extends TableInfo implements Serializable { public static final String PARTITIONED_JOIN_KEY = "partitionedJoin"; - public static final String CACHE_MODE_KEY = "cacheMode"; - - public static final String ASYNC_CAP_KEY = "asyncCapacity"; - - public static final String ASYNC_TIMEOUT_KEY = "asyncTimeout"; - private String cacheType = "none";//None or LRU or ALL private int cacheSize = 10000; private long cacheTimeout = 60 * 1000;// - private int asyncCapacity=100; - - private int asyncTimeout=10000; - private boolean partitionedJoin = false; - private String cacheMode="ordered"; - public RowTypeInfo getRowTypeInfo(){ Class[] fieldClass = getFieldClasses(); TypeInformation[] types = new TypeInformation[fieldClass.length]; @@ -107,28 +95,4 @@ public boolean isPartitionedJoin() { public void setPartitionedJoin(boolean partitionedJoin) { this.partitionedJoin = partitionedJoin; } - - public String getCacheMode() { - return cacheMode; - } - - public void setCacheMode(String cacheMode) { - this.cacheMode = cacheMode; - } - - public int getAsyncCapacity() { - return asyncCapacity; - } - - public void setAsyncCapacity(int asyncCapacity) { - this.asyncCapacity = asyncCapacity; - } - - public int getAsyncTimeout() { - return asyncTimeout; - } - - public void setAsyncTimeout(int asyncTimeout) { - this.asyncTimeout = asyncTimeout; - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java index 185911c7e..4eb78b4c4 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/StreamSideFactory.java @@ -16,10 +16,11 @@ * limitations under the License. */ + package com.dtstack.flink.sql.side; -import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.classloader.DtClassLoader; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.table.AbsSideTableParser; import com.dtstack.flink.sql.table.AbsTableParser; @@ -29,7 +30,6 @@ * get specify side parser * Date: 2018/7/25 * Company: www.dtstack.com - * * @author xuchao */ @@ -40,15 +40,18 @@ public class StreamSideFactory { public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir, String cacheType) throws Exception { String sideOperator = ECacheType.ALL.name().equals(cacheType) ? "all" : "async"; + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); String pluginJarPath = PluginUtil.getSideJarFileDirPath(pluginType, sideOperator, "side", sqlRootDir); + + DtClassLoader dtClassLoader = (DtClassLoader) classLoader; + PluginUtil.addPluginJar(pluginJarPath, dtClassLoader); String className = PluginUtil.getSqlParserClassName(pluginType, CURR_TYPE); - return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { - Class sideParser = cl.loadClass(className); - if (!AbsSideTableParser.class.isAssignableFrom(sideParser)) { - throw new RuntimeException("class " + sideParser.getName() + " not subClass of AbsSideTableParser"); - } - return sideParser.asSubclass(AbsTableParser.class).newInstance(); - }); + Class sideParser = dtClassLoader.loadClass(className); + if(!AbsSideTableParser.class.isAssignableFrom(sideParser)){ + throw new RuntimeException("class " + sideParser.getName() + " not subClass of AbsSideTableParser"); + } + + return sideParser.asSubclass(AbsTableParser.class).newInstance(); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java index 700e13bb2..475536a4c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/cache/LRUSideCache.java @@ -21,8 +21,8 @@ package com.dtstack.flink.sql.side.cache; import com.dtstack.flink.sql.side.SideTableInfo; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; +import org.apache.flink.calcite.shaded.com.google.common.cache.Cache; +import org.apache.flink.calcite.shaded.com.google.common.cache.CacheBuilder; import java.util.concurrent.TimeUnit; diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java index 290804200..df9a1c175 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideAsyncOperator.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.operator; -import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.classloader.DtClassLoader; import com.dtstack.flink.sql.side.AsyncReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; @@ -44,34 +44,26 @@ public class SideAsyncOperator { private static final String PATH_FORMAT = "%sasyncside"; - private static final String OPERATOR_TYPE = "Async"; - - private static final String ORDERED = "ordered"; - + //TODO need to set by create table task + private static int asyncCapacity = 100; private static AsyncReqRow loadAsyncReq(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); - String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); - return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> - cl.loadClass(className).asSubclass(AsyncReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) - .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + DtClassLoader dtClassLoader = (DtClassLoader) classLoader; + PluginUtil.addPluginJar(pluginJarPath, dtClassLoader); + String className = PluginUtil.getSqlSideClassName(sideType, "side", "Async"); + return dtClassLoader.loadClass(className).asSubclass(AsyncReqRow.class) + .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class).newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { AsyncReqRow asyncDbReq = loadAsyncReq(sideType, sqlRootDir, rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); - //TODO How much should be set for the degree of parallelism? Timeout? capacity settings? - if (ORDERED.equals(sideTableInfo.getCacheMode())){ - return AsyncDataStream.orderedWait(inputStream, asyncDbReq, sideTableInfo.getAsyncTimeout(), TimeUnit.MILLISECONDS, sideTableInfo.getAsyncCapacity()) - .setParallelism(sideTableInfo.getParallelism()); - }else { - return AsyncDataStream.unorderedWait(inputStream, asyncDbReq, sideTableInfo.getAsyncTimeout(), TimeUnit.MILLISECONDS, sideTableInfo.getAsyncCapacity()) - .setParallelism(sideTableInfo.getParallelism()); - } - + return AsyncDataStream.orderedWait(inputStream, asyncDbReq, 10000, TimeUnit.MILLISECONDS, asyncCapacity) + .setParallelism(sideTableInfo.getParallelism()); } } diff --git a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java index 5aa810b0f..72a67d00b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java +++ b/core/src/main/java/com/dtstack/flink/sql/side/operator/SideWithAllCacheOperator.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.operator; -import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.classloader.DtClassLoader; import com.dtstack.flink.sql.side.AllReqRow; import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.JoinInfo; @@ -28,6 +28,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; +import java.net.MalformedURLException; import java.util.List; /** @@ -42,19 +43,22 @@ public class SideWithAllCacheOperator { private static final String PATH_FORMAT = "%sallside"; - private static final String OPERATOR_TYPE = "All"; - private static AllReqRow loadFlatMap(String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) throws Exception { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); String pathOfType = String.format(PATH_FORMAT, sideType); String pluginJarPath = PluginUtil.getJarFileDirPath(pathOfType, sqlRootDir); - String className = PluginUtil.getSqlSideClassName(sideType, "side", OPERATOR_TYPE); - return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> cl.loadClass(className).asSubclass(AllReqRow.class) - .getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) - .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + DtClassLoader dtClassLoader = (DtClassLoader) classLoader; + PluginUtil.addPluginJar(pluginJarPath, dtClassLoader); + String className = PluginUtil.getSqlSideClassName(sideType, "side", "All"); + + return dtClassLoader.loadClass(className).asSubclass(AllReqRow.class).getConstructor(RowTypeInfo.class, JoinInfo.class, List.class, SideTableInfo.class) + .newInstance(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); + + } public static DataStream getSideJoinDataStream(DataStream inputStream, String sideType, String sqlRootDir, RowTypeInfo rowTypeInfo, JoinInfo joinInfo, diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/MetricOutputFormat.java b/core/src/main/java/com/dtstack/flink/sql/sink/MetricOutputFormat.java index f56f531e0..4e11280e7 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/MetricOutputFormat.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/MetricOutputFormat.java @@ -32,13 +32,10 @@ public abstract class MetricOutputFormat extends RichOutputFormat{ protected transient Counter outRecords; - protected transient Counter outDirtyRecords; - protected transient Meter outRecordsRate; public void initMetric() { outRecords = getRuntimeContext().getMetricGroup().counter(MetricConstant.DT_NUM_RECORDS_OUT); - outDirtyRecords = getRuntimeContext().getMetricGroup().counter(MetricConstant.DT_NUM_DIRTY_RECORDS_OUT); outRecordsRate = getRuntimeContext().getMetricGroup().meter(MetricConstant.DT_NUM_RECORDS_OUT_RATE, new MeterView(outRecords, 20)); } diff --git a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java index 53460081d..ebda80c8a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/sink/StreamSinkFactory.java @@ -20,11 +20,12 @@ package com.dtstack.flink.sql.sink; -import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.classloader.DtClassLoader; import com.dtstack.flink.sql.table.AbsTableParser; import com.dtstack.flink.sql.table.TargetTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import org.apache.flink.table.sinks.TableSink; /** @@ -41,33 +42,51 @@ public class StreamSinkFactory { private static final String DIR_NAME_FORMAT = "%ssink"; public static AbsTableParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + if(!(classLoader instanceof DtClassLoader)){ + throw new RuntimeException("it's not a correct classLoader instance, it's type must be DtClassLoader!"); + } + + DtClassLoader dtClassLoader = (DtClassLoader) classLoader; + String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); + PluginUtil.addPluginJar(pluginJarPath, dtClassLoader); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); + Class targetParser = dtClassLoader.loadClass(className); - return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { - Class targetParser = cl.loadClass(className); - if(!AbsTableParser.class.isAssignableFrom(targetParser)){ - throw new RuntimeException("class " + targetParser.getName() + " not subClass of AbsTableParser"); - } - return targetParser.asSubclass(AbsTableParser.class).newInstance(); - }); + if(!AbsTableParser.class.isAssignableFrom(targetParser)){ + throw new RuntimeException("class " + targetParser.getName() + " not subClass of AbsTableParser"); + } + + return targetParser.asSubclass(AbsTableParser.class).newInstance(); } public static TableSink getTableSink(TargetTableInfo targetTableInfo, String localSqlRootDir) throws Exception { + + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + if(!(classLoader instanceof DtClassLoader)){ + throw new RuntimeException("it's not a correct classLoader instance, it's type must be DtClassLoader!"); + } + + DtClassLoader dtClassLoader = (DtClassLoader) classLoader; + String pluginType = targetTableInfo.getType(); String pluginJarDirPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), localSqlRootDir); + + PluginUtil.addPluginJar(pluginJarDirPath, dtClassLoader); + String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getGenerClassName(typeNoVersion, CURR_TYPE); + Class sinkClass = dtClassLoader.loadClass(className); + + if(!IStreamSinkGener.class.isAssignableFrom(sinkClass)){ + throw new RuntimeException("class " + sinkClass + " not subClass of IStreamSinkGener"); + } - return ClassLoaderManager.newInstance(pluginJarDirPath, (cl) -> { - Class sinkClass = cl.loadClass(className); - if(!IStreamSinkGener.class.isAssignableFrom(sinkClass)){ - throw new RuntimeException("class " + sinkClass + " not subClass of IStreamSinkGener"); - } - IStreamSinkGener streamSinkGener = sinkClass.asSubclass(IStreamSinkGener.class).newInstance(); - Object result = streamSinkGener.genStreamSink(targetTableInfo); - return (TableSink) result; - }); + IStreamSinkGener streamSinkGener = sinkClass.asSubclass(IStreamSinkGener.class).newInstance(); + Object result = streamSinkGener.genStreamSink(targetTableInfo); + return (TableSink) result; } } diff --git a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java index 1057fb0ed..b8dfe66e2 100644 --- a/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/source/StreamSourceFactory.java @@ -21,11 +21,12 @@ package com.dtstack.flink.sql.source; -import com.dtstack.flink.sql.classloader.ClassLoaderManager; +import com.dtstack.flink.sql.classloader.DtClassLoader; import com.dtstack.flink.sql.table.AbsSourceParser; import com.dtstack.flink.sql.table.SourceTableInfo; import com.dtstack.flink.sql.util.DtStringUtil; import com.dtstack.flink.sql.util.PluginUtil; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; @@ -45,16 +46,21 @@ public class StreamSourceFactory { public static AbsSourceParser getSqlParser(String pluginType, String sqlRootDir) throws Exception { + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, pluginType), sqlRootDir); + + DtClassLoader dtClassLoader = (DtClassLoader) classLoader; + PluginUtil.addPluginJar(pluginJarPath, dtClassLoader); + String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(pluginType); String className = PluginUtil.getSqlParserClassName(typeNoVersion, CURR_TYPE); - return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { - Class sourceParser = cl.loadClass(className); - if(!AbsSourceParser.class.isAssignableFrom(sourceParser)){ - throw new RuntimeException("class " + sourceParser.getName() + " not subClass of AbsSourceParser"); - } - return sourceParser.asSubclass(AbsSourceParser.class).newInstance(); - }); + Class sourceParser = dtClassLoader.loadClass(className); + if(!AbsSourceParser.class.isAssignableFrom(sourceParser)){ + throw new RuntimeException("class " + sourceParser.getName() + " not subClass of AbsSourceParser"); + } + + return sourceParser.asSubclass(AbsSourceParser.class).newInstance(); } /** @@ -67,17 +73,21 @@ public static Table getStreamSource(SourceTableInfo sourceTableInfo, StreamExecu String sourceTypeStr = sourceTableInfo.getType(); String typeNoVersion = DtStringUtil.getPluginTypeWithoutVersion(sourceTypeStr); + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + String pluginJarPath = PluginUtil.getJarFileDirPath(String.format(DIR_NAME_FORMAT, sourceTypeStr), sqlRootDir); String className = PluginUtil.getGenerClassName(typeNoVersion, CURR_TYPE); - return ClassLoaderManager.newInstance(pluginJarPath, (cl) -> { - Class sourceClass = cl.loadClass(className); - if(!IStreamSourceGener.class.isAssignableFrom(sourceClass)){ - throw new RuntimeException("class " + sourceClass.getName() + " not subClass of IStreamSourceGener"); - } + DtClassLoader dtClassLoader = (DtClassLoader) classLoader; + PluginUtil.addPluginJar(pluginJarPath, dtClassLoader); + Class sourceClass = dtClassLoader.loadClass(className); + + if(!IStreamSourceGener.class.isAssignableFrom(sourceClass)){ + throw new RuntimeException("class " + sourceClass.getName() + " not subClass of IStreamSourceGener"); + } - IStreamSourceGener sourceGener = sourceClass.asSubclass(IStreamSourceGener.class).newInstance(); - return (Table) sourceGener.genStreamSource(sourceTableInfo, env, tableEnv); - }); + IStreamSourceGener sourceGener = sourceClass.asSubclass(IStreamSourceGener.class).newInstance(); + Object object = sourceGener.genStreamSource(sourceTableInfo, env, tableEnv); + return (Table) object; } } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java index 3c4199c8c..f8ede801b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSideTableParser.java @@ -16,13 +16,14 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.table; import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.util.MathUtil; + import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -84,31 +85,6 @@ protected void parseCacheProp(SideTableInfo sideTableInfo, Map p sideTableInfo.setPartitionedJoin(true); } } - - if(props.containsKey(SideTableInfo.CACHE_MODE_KEY.toLowerCase())){ - String cachemode = MathUtil.getString(props.get(SideTableInfo.CACHE_MODE_KEY.toLowerCase())); - - if(!cachemode.equalsIgnoreCase("ordered") && !cachemode.equalsIgnoreCase("unordered")){ - throw new RuntimeException("cachemode must ordered or unordered!"); - } - sideTableInfo.setCacheMode(cachemode.toLowerCase()); - } - - if(props.containsKey(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())){ - Integer asyncCap = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_CAP_KEY.toLowerCase())); - if(asyncCap < 0){ - throw new RuntimeException("asyncCapacity size need > 0."); - } - sideTableInfo.setAsyncCapacity(asyncCap); - } - - if(props.containsKey(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())){ - Integer asyncTimeout = MathUtil.getIntegerVal(props.get(SideTableInfo.ASYNC_TIMEOUT_KEY.toLowerCase())); - if (asyncTimeout<0){ - throw new RuntimeException("asyncTimeout size need > 0."); - } - sideTableInfo.setAsyncTimeout(asyncTimeout); - } } } } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java index 82000b386..1b676ff8d 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsSourceParser.java @@ -20,7 +20,6 @@ package com.dtstack.flink.sql.table; -import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.MathUtil; import java.util.regex.Matcher; @@ -37,21 +36,19 @@ public abstract class AbsSourceParser extends AbsTableParser { private static final String VIRTUAL_KEY = "virtualFieldKey"; + private static final String WATERMARK_KEY = "waterMarkKey"; - private static final String NOTNULL_KEY = "notNullKey"; private static Pattern virtualFieldKeyPattern = Pattern.compile("(?i)^(\\S+\\([^\\)]+\\))\\s+AS\\s+(\\w+)$"); + private static Pattern waterMarkKeyPattern = Pattern.compile("(?i)^\\s*WATERMARK\\s+FOR\\s+(\\S+)\\s+AS\\s+withOffset\\(\\s*(\\S+)\\s*,\\s*(\\d+)\\s*\\)$"); - private static Pattern notNullKeyPattern = Pattern.compile("(?i)^(\\w+)\\s+(\\w+)\\s+NOT\\s+NULL?$"); static { keyPatternMap.put(VIRTUAL_KEY, virtualFieldKeyPattern); keyPatternMap.put(WATERMARK_KEY, waterMarkKeyPattern); - keyPatternMap.put(NOTNULL_KEY, notNullKeyPattern); keyHandlerMap.put(VIRTUAL_KEY, AbsSourceParser::dealVirtualField); keyHandlerMap.put(WATERMARK_KEY, AbsSourceParser::dealWaterMark); - keyHandlerMap.put(NOTNULL_KEY, AbsSourceParser::dealNotNull); } static void dealVirtualField(Matcher matcher, TableInfo tableInfo){ @@ -69,18 +66,4 @@ static void dealWaterMark(Matcher matcher, TableInfo tableInfo){ sourceTableInfo.setEventTimeField(eventTimeField); sourceTableInfo.setMaxOutOrderness(offset); } - - static void dealNotNull(Matcher matcher, TableInfo tableInfo) { - String fieldName = matcher.group(1); - String fieldType = matcher.group(2); - Class fieldClass= ClassUtil.stringConvertClass(fieldType); - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); - fieldExtraInfo.setNotNull(true); - - tableInfo.addPhysicalMappings(fieldName, fieldName); - tableInfo.addField(fieldName); - tableInfo.addFieldClass(fieldClass); - tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(fieldExtraInfo); - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java index aa0639c8a..ddbf269e9 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/AbsTableParser.java @@ -22,8 +22,8 @@ import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.DtStringUtil; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; +import org.apache.flink.shaded.curator.org.apache.curator.shaded.com.google.common.collect.Maps; import java.util.List; import java.util.Map; @@ -79,32 +79,30 @@ public boolean dealKeyPattern(String fieldRow, TableInfo tableInfo){ public void parseFieldsInfo(String fieldsInfo, TableInfo tableInfo){ - List fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ','); + String[] fieldRows = DtStringUtil.splitIgnoreQuotaBrackets(fieldsInfo, ","); for(String fieldRow : fieldRows){ fieldRow = fieldRow.trim(); - - String[] filedInfoArr = fieldRow.split("\\s+"); - if(filedInfoArr.length < 2 ){ - throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); - } - boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo); + if(isMatcherKey){ continue; } + String[] filedInfoArr = fieldRow.split("\\s+"); + if(filedInfoArr.length < 2){ + throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow)); + } + //Compatible situation may arise in space in the fieldName String[] filedNameArr = new String[filedInfoArr.length - 1]; System.arraycopy(filedInfoArr, 0, filedNameArr, 0, filedInfoArr.length - 1); String fieldName = String.join(" ", filedNameArr); String fieldType = filedInfoArr[filedInfoArr.length - 1 ].trim(); - Class fieldClass = dbTypeConvertToJavaType(fieldType); + Class fieldClass = ClassUtil.stringConvertClass(fieldType); - tableInfo.addPhysicalMappings(filedInfoArr[0],filedInfoArr[0]); tableInfo.addField(fieldName); tableInfo.addFieldClass(fieldClass); tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(null); } tableInfo.finish(); @@ -116,9 +114,4 @@ public static void dealPrimaryKey(Matcher matcher, TableInfo tableInfo){ List primaryKes = Lists.newArrayList(splitArry); tableInfo.setPrimaryKeys(primaryKes); } - - public Class dbTypeConvertToJavaType(String fieldType) { - return ClassUtil.stringConvertClass(fieldType); - } - } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java index 9a41fa0a1..a92aa9fb1 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/SourceTableInfo.java @@ -20,9 +20,9 @@ package com.dtstack.flink.sql.table; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.base.Strings; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Maps; import org.apache.flink.util.StringUtils; import java.util.ArrayList; @@ -71,7 +71,6 @@ public void setMaxOutOrderness(Integer maxOutOrderness) { if(maxOutOrderness == null){ return; } - this.maxOutOrderness = maxOutOrderness; } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java index a2454b893..57f94e2e0 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfo.java @@ -20,8 +20,8 @@ package com.dtstack.flink.sql.table; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; import java.io.Serializable; import java.util.List; @@ -57,8 +57,6 @@ public abstract class TableInfo implements Serializable { private final List fieldClassList = Lists.newArrayList(); - private final List fieldExtraInfoList = Lists.newArrayList(); - private List primaryKeys; private Integer parallelism = 1; @@ -145,10 +143,6 @@ public void setFieldClasses(Class[] fieldClasses) { this.fieldClasses = fieldClasses; } - public void addFieldExtraInfo(FieldExtraInfo extraInfo) { - fieldExtraInfoList.add(extraInfo); - } - public List getFieldList() { return fieldList; } @@ -165,10 +159,6 @@ public Map getPhysicalFields() { return physicalFields; } - public List getFieldExtraInfoList() { - return fieldExtraInfoList; - } - public void setPhysicalFields(Map physicalFields) { this.physicalFields = physicalFields; } @@ -178,25 +168,4 @@ public void finish(){ this.fieldClasses = fieldClassList.toArray(new Class[fieldClassList.size()]); this.fieldTypes = fieldTypeList.toArray(new String[fieldTypeList.size()]); } - - /** - * field extra info，used to store `not null` `default 0`...， - * - * now, only support not null - */ - public static class FieldExtraInfo implements Serializable { - - /** - * default false：allow field is null - */ - boolean notNull = false; - - public boolean getNotNull() { - return notNull; - } - - public void setNotNull(boolean notNull) { - this.notNull = notNull; - } - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java index ae98d90ae..b3a07d6d5 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TableInfoParser.java @@ -20,6 +20,7 @@ package com.dtstack.flink.sql.table; +import com.dtstack.flink.sql.enums.ECacheType; import com.dtstack.flink.sql.enums.ETableType; import com.dtstack.flink.sql.parser.CreateTableParser; import com.dtstack.flink.sql.side.SideTableInfo; @@ -27,8 +28,8 @@ import com.dtstack.flink.sql.sink.StreamSinkFactory; import com.dtstack.flink.sql.source.StreamSourceFactory; import com.dtstack.flink.sql.util.MathUtil; -import com.google.common.base.Strings; -import com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.base.Strings; +import org.apache.flink.shaded.curator.org.apache.curator.shaded.com.google.common.collect.Maps; import java.util.Map; import java.util.regex.Matcher; diff --git a/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java b/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java index 70b625b5c..a9a846707 100644 --- a/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java +++ b/core/src/main/java/com/dtstack/flink/sql/table/TargetTableInfo.java @@ -30,16 +30,4 @@ public abstract class TargetTableInfo extends TableInfo { public static final String TARGET_SUFFIX = "Sink"; - - public static final String SINK_DATA_TYPE = "sinkdatatype"; - - private String sinkDataType = "json"; - - public String getSinkDataType() { - return sinkDataType; - } - - public void setSinkDataType(String sinkDataType) { - this.sinkDataType = sinkDataType; - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/factory/DTThreadFactory.java b/core/src/main/java/com/dtstack/flink/sql/threadFactory/DTThreadFactory.java similarity index 97% rename from core/src/main/java/com/dtstack/flink/sql/factory/DTThreadFactory.java rename to core/src/main/java/com/dtstack/flink/sql/threadFactory/DTThreadFactory.java index e9f9dfa64..1c16581ef 100644 --- a/core/src/main/java/com/dtstack/flink/sql/factory/DTThreadFactory.java +++ b/core/src/main/java/com/dtstack/flink/sql/threadFactory/DTThreadFactory.java @@ -18,7 +18,7 @@ -package com.dtstack.flink.sql.factory; +package com.dtstack.flink.sql.threadFactory; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java index feebbe3f1..4012e5c3b 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java @@ -22,7 +22,6 @@ import java.math.BigDecimal; import java.sql.Date; -import java.sql.Time; import java.sql.Timestamp; /** @@ -36,55 +35,38 @@ public class ClassUtil { public static Class stringConvertClass(String str) { switch (str.toLowerCase()) { case "boolean": - case "bit": return Boolean.class; - case "smallint": - case "smallintunsigned": - case "tinyint": - case "tinyintunsigned": - case "mediumint": - case "mediumintunsigned": case "integer": case "int": return Integer.class; - case "blob": - return Byte.class; - case "bigint": - case "intunsigned": - case "integerunsigned": - case "bigintunsigned": return Long.class; + case "tinyint": + return Byte.class; + + case "smallint": + return Short.class; + case "varchar": - case "char": - case "text": return String.class; case "real": case "float": - case "realunsigned": - case "floatunsigned": return Float.class; case "double": - case "doubleunsigned": return Double.class; case "date": return Date.class; - case "datetime": case "timestamp": return Timestamp.class; - case "time": - return Time.class; - case "decimal": - case "decimalunsigned": return BigDecimal.class; } diff --git a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java b/core/src/main/java/com/dtstack/flink/sql/util/ConfigConstrant.java similarity index 79% rename from core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java rename to core/src/main/java/com/dtstack/flink/sql/util/ConfigConstrant.java index 76f5996c3..160d8f411 100644 --- a/core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/ConfigConstrant.java @@ -18,7 +18,7 @@ -package com.dtstack.flink.sql.constrant; +package com.dtstack.flink.sql.util; /** @@ -29,9 +29,7 @@ */ public class ConfigConstrant { - public static final String SQL_CHECKPOINT_INTERVAL_KEY = "sql.checkpoint.interval"; - // 兼容上层 - public static final String FLINK_CHECKPOINT_INTERVAL_KEY = "flink.checkpoint.interval"; + public static final String FLINK_CHECKPOINT_INTERVAL_KEY = "sql.checkpoint.interval"; public static final String FLINK_CHECKPOINT_MODE_KEY = "sql.checkpoint.mode"; @@ -39,11 +37,7 @@ public class ConfigConstrant { public static final String FLINK_MAXCONCURRENTCHECKPOINTS_KEY = "sql.max.concurrent.checkpoints"; - public static final String SQL_CHECKPOINT_CLEANUPMODE_KEY = "sql.checkpoint.cleanup.mode"; - - public static final String FLINK_CHECKPOINT_CLEANUPMODE_KEY = "flink.checkpoint.cleanup.mode"; - - + public static final String FLINK_CHECKPOINT_CLEANUPMODE_KEY = "sql.checkpoint.cleanup.mode"; public static final String FLINK_CHECKPOINT_DATAURI_KEY = "flinkCheckpointDataURI"; @@ -57,11 +51,4 @@ public class ConfigConstrant { public static final String FLINK_TIME_CHARACTERISTIC_KEY = "time.characteristic"; - // restart plocy - public static final int failureRate = 3; - - public static final int failureInterval = 6; //min - - public static final int delayInterval = 10; //sec - } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java index 862b0700e..959d4b13c 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/DtStringUtil.java @@ -16,16 +16,16 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.util; import com.dtstack.flink.sql.enums.ColumnType; import org.apache.commons.lang3.StringUtils; -import com.google.common.base.Strings; -import com.google.common.collect.Maps; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.sql.Timestamp; +import org.apache.flink.calcite.shaded.com.google.common.base.Strings; +import org.apache.flink.calcite.shaded.com.google.common.collect.Maps; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -57,42 +57,26 @@ public static List splitIgnoreQuota(String str, char delimiter){ List tokensList = new ArrayList<>(); boolean inQuotes = false; boolean inSingleQuotes = false; - int bracketLeftNum = 0; StringBuilder b = new StringBuilder(); - char[] chars = str.toCharArray(); - int idx = 0; - for (char c : chars) { - char flag = 0; - if (idx > 0) { - flag = chars[idx - 1]; - } - if (c == delimiter) { + for (char c : str.toCharArray()) { + if(c == delimiter){ if (inQuotes) { b.append(c); - } else if (inSingleQuotes) { - b.append(c); - } else if (bracketLeftNum > 0) { + } else if(inSingleQuotes){ b.append(c); - } else { + }else { tokensList.add(b.toString()); b = new StringBuilder(); } - } else if (c == '\"' && '\\' != flag && !inSingleQuotes) { + }else if(c == '\"'){ inQuotes = !inQuotes; b.append(c); - } else if (c == '\'' && '\\' != flag && !inQuotes) { + }else if(c == '\''){ inSingleQuotes = !inSingleQuotes; b.append(c); - } else if (c == '(' && !inSingleQuotes && !inQuotes) { - bracketLeftNum++; - b.append(c); - } else if (c == ')' && !inSingleQuotes && !inQuotes) { - bracketLeftNum--; - b.append(c); - } else { + }else{ b.append(c); } - idx++; } tokensList.add(b.toString()); @@ -100,6 +84,16 @@ public static List splitIgnoreQuota(String str, char delimiter){ return tokensList; } + /*** + * Split the specified string delimiter --- ignored in brackets and quotation marks delimiter + * @param str + * @param delimter + * @return + */ + public static String[] splitIgnoreQuotaBrackets(String str, String delimter){ + String splitPatternStr = delimter + "(?![^()]*+\\))(?![^{}]*+})(?![^\\[\\]]*+\\])(?=(?:[^\"]|\"[^\"]*\")*$)"; + return str.split(splitPatternStr); + } public static String replaceIgnoreQuota(String str, String oriStr, String replaceStr){ String splitPatternStr = oriStr + "(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)(?=(?:[^']*'[^']*')*[^']*$)"; @@ -231,54 +225,4 @@ public static boolean isJosn(String str){ } return flag; } - - public static Object parse(String str,Class clazz){ - String fieldType = clazz.getName(); - Object object = null; - if(fieldType.equals(Integer.class.getName())){ - object = Integer.parseInt(str); - }else if(fieldType.equals(Long.class.getName())){ - object = Long.parseLong(str); - }else if(fieldType.equals(Byte.class.getName())){ - object = str.getBytes()[0]; - }else if(fieldType.equals(String.class.getName())){ - object = str; - }else if(fieldType.equals(Float.class.getName())){ - object = Float.parseFloat(str); - }else if(fieldType.equals(Double.class.getName())){ - object = Double.parseDouble(str); - }else if (fieldType.equals(Timestamp.class.getName())){ - object = Timestamp.valueOf(str); - }else{ - throw new RuntimeException("no support field type for sql. the input type:" + fieldType); - } - return object; - } - - - public static String firstUpperCase(String str) { - return str.substring(0, 1).toUpperCase() + str.substring(1); - } - - public static String getTableFullPath(String schema, String tableName) { - if (StringUtils.isEmpty(schema)){ - return addQuoteForStr(tableName); - } - String schemaAndTabName = addQuoteForStr(schema) + "." + addQuoteForStr(tableName); - return schemaAndTabName; - } - - - - public static String addQuoteForStr(String column) { - return getStartQuote() + column + getEndQuote(); - } - - public static String getStartQuote() { - return "\""; - } - - public static String getEndQuote() { - return "\""; - } } diff --git a/core/src/main/java/com/dtstack/flink/sql/util/FlinkUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/FlinkUtil.java index 5811105e4..0ba59ace6 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/FlinkUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/FlinkUtil.java @@ -16,13 +16,11 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.util; -import com.dtstack.flink.sql.classloader.ClassLoaderManager; -import com.dtstack.flink.sql.constrant.ConfigConstrant; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.runtime.state.filesystem.FsStateBackend; @@ -35,8 +33,6 @@ import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.table.functions.ScalarFunction; import org.apache.flink.table.functions.TableFunction; -import org.apache.flink.table.functions.AggregateFunction; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,14 +67,12 @@ public static void openCheckpoint(StreamExecutionEnvironment env, Properties pro } //设置了时间间隔才表明开启了checkpoint - if(properties.getProperty(ConfigConstrant.SQL_CHECKPOINT_INTERVAL_KEY) == null && properties.getProperty(ConfigConstrant.FLINK_CHECKPOINT_INTERVAL_KEY) == null){ + if(properties.getProperty(ConfigConstrant.FLINK_CHECKPOINT_INTERVAL_KEY) == null){ return; }else{ - Long sql_interval = Long.valueOf(properties.getProperty(ConfigConstrant.SQL_CHECKPOINT_INTERVAL_KEY,"0")); - Long flink_interval = Long.valueOf(properties.getProperty(ConfigConstrant.FLINK_CHECKPOINT_INTERVAL_KEY, "0")); - long checkpointInterval = Math.max(sql_interval, flink_interval); + Long interval = Long.valueOf(properties.getProperty(ConfigConstrant.FLINK_CHECKPOINT_INTERVAL_KEY)); //start checkpoint every ${interval} - env.enableCheckpointing(checkpointInterval); + env.enableCheckpointing(interval); } String checkMode = properties.getProperty(ConfigConstrant.FLINK_CHECKPOINT_MODE_KEY); @@ -106,22 +100,17 @@ public static void openCheckpoint(StreamExecutionEnvironment env, Properties pro env.getCheckpointConfig().setMaxConcurrentCheckpoints(maxConcurrCheckpoints); } - Boolean sqlCleanMode = MathUtil.getBoolean(properties.getProperty(ConfigConstrant.SQL_CHECKPOINT_CLEANUPMODE_KEY), false); - Boolean flinkCleanMode = MathUtil.getBoolean(properties.getProperty(ConfigConstrant.FLINK_CHECKPOINT_CLEANUPMODE_KEY), false); - - String cleanupModeStr = "false"; - if (sqlCleanMode || flinkCleanMode ){ - cleanupModeStr = "true"; - } - - if ("true".equalsIgnoreCase(cleanupModeStr)){ - env.getCheckpointConfig().enableExternalizedCheckpoints( - CheckpointConfig.ExternalizedCheckpointCleanup.DELETE_ON_CANCELLATION); - } else if("false".equalsIgnoreCase(cleanupModeStr) || cleanupModeStr == null){ - env.getCheckpointConfig().enableExternalizedCheckpoints( - CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); - } else{ - throw new RuntimeException("not support value of cleanup mode :" + cleanupModeStr); + String cleanupModeStr = properties.getProperty(ConfigConstrant.FLINK_CHECKPOINT_CLEANUPMODE_KEY); + if(cleanupModeStr != null){//设置在cancel job情况下checkpoint是否被保存 + if("true".equalsIgnoreCase(cleanupModeStr)){ + env.getCheckpointConfig().enableExternalizedCheckpoints( + CheckpointConfig.ExternalizedCheckpointCleanup.DELETE_ON_CANCELLATION); + }else if("false".equalsIgnoreCase(cleanupModeStr)){ + env.getCheckpointConfig().enableExternalizedCheckpoints( + CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); + }else{ + throw new RuntimeException("not support value of cleanup mode :" + cleanupModeStr); + } } String backendPath = properties.getProperty(ConfigConstrant.FLINK_CHECKPOINT_DATAURI_KEY); @@ -149,7 +138,6 @@ public static void setStreamTimeCharacteristic(StreamExecutionEnvironment env, P if(characteristicStr.equalsIgnoreCase(tmp.toString())){ env.setStreamTimeCharacteristic(tmp); flag = true; - break; } } @@ -159,19 +147,20 @@ public static void setStreamTimeCharacteristic(StreamExecutionEnvironment env, P } + /** - * TABLE|SCALA|AGGREGATE + * FIXME 暂时不支持 UDF 实现类--有参构造方法 + * TABLE|SCALA * 注册UDF到table env */ - public static void registerUDF(String type, String classPath, String funcName, TableEnvironment tableEnv, ClassLoader classLoader){ + public static void registerUDF(String type, String classPath, String funcName, TableEnvironment tableEnv, + ClassLoader classLoader){ if("SCALA".equalsIgnoreCase(type)){ registerScalaUDF(classPath, funcName, tableEnv, classLoader); }else if("TABLE".equalsIgnoreCase(type)){ registerTableUDF(classPath, funcName, tableEnv, classLoader); - }else if("AGGREGATE".equalsIgnoreCase(type)){ - registerAggregateUDF(classPath, funcName, tableEnv, classLoader); }else{ - throw new RuntimeException("not support of UDF which is not in (TABLE, SCALA, AGGREGATE)"); + throw new RuntimeException("not support of UDF which is not in (TABLE, SCALA)"); } } @@ -182,7 +171,8 @@ public static void registerUDF(String type, String classPath, String funcName, T * @param funcName * @param tableEnv */ - public static void registerScalaUDF(String classPath, String funcName, TableEnvironment tableEnv, ClassLoader classLoader){ + public static void registerScalaUDF(String classPath, String funcName, TableEnvironment tableEnv, + ClassLoader classLoader){ try{ ScalarFunction udfFunc = Class.forName(classPath, false, classLoader) .asSubclass(ScalarFunction.class).newInstance(); @@ -190,21 +180,23 @@ public static void registerScalaUDF(String classPath, String funcName, TableEnvi logger.info("register scala function:{} success.", funcName); }catch (Exception e){ logger.error("", e); - throw new RuntimeException("register UDF exception:", e); + throw new RuntimeException("register UDF exception:" + e.getMessage()); } } /** * 注册自定义TABLEFFUNC方法到env上 - * + * TODO 对User-Defined Aggregate Functions的支持 * @param classPath * @param funcName * @param tableEnv */ - public static void registerTableUDF(String classPath, String funcName, TableEnvironment tableEnv, ClassLoader classLoader){ + public static void registerTableUDF(String classPath, String funcName, TableEnvironment tableEnv, + ClassLoader classLoader){ try { TableFunction udfFunc = Class.forName(classPath, false, classLoader) .asSubclass(TableFunction.class).newInstance(); + if(tableEnv instanceof StreamTableEnvironment){ ((StreamTableEnvironment)tableEnv).registerFunction(funcName, udfFunc); }else if(tableEnv instanceof BatchTableEnvironment){ @@ -216,35 +208,10 @@ public static void registerTableUDF(String classPath, String funcName, TableEnvi logger.info("register table function:{} success.", funcName); }catch (Exception e){ logger.error("", e); - throw new RuntimeException("register Table UDF exception:", e); + throw new RuntimeException("register Table UDF exception:" + e.getMessage()); } } - /** - * 注册自定义Aggregate FUNC方法到env上 - * - * @param classPath - * @param funcName - * @param tableEnv - */ - public static void registerAggregateUDF(String classPath, String funcName, TableEnvironment tableEnv, ClassLoader classLoader) { - try { - AggregateFunction udfFunc = Class.forName(classPath, false, classLoader) - .asSubclass(AggregateFunction.class).newInstance(); - if (tableEnv instanceof StreamTableEnvironment) { - ((StreamTableEnvironment) tableEnv).registerFunction(funcName, udfFunc); - } else if (tableEnv instanceof BatchTableEnvironment) { - ((BatchTableEnvironment) tableEnv).registerFunction(funcName, udfFunc); - } else { - throw new RuntimeException("no support tableEnvironment class for " + tableEnv.getClass().getName()); - } - - logger.info("register Aggregate function:{} success.", funcName); - } catch (Exception e) { - logger.error("", e); - throw new RuntimeException("register Aggregate UDF exception:", e); - } - } /** * @@ -279,9 +246,21 @@ public static long getBufferTimeoutMillis(Properties properties){ } public static URLClassLoader loadExtraJar(List jarURLList, URLClassLoader classLoader) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + + int size = 0; + for(URL url : jarURLList){ + if(url.toString().endsWith(".jar")){ + size++; + } + } + + URL[] urlArray = new URL[size]; + int i=0; for(URL url : jarURLList){ if(url.toString().endsWith(".jar")){ + urlArray[i] = url; urlClassLoaderAddUrl(classLoader, url); + i++; } } @@ -309,4 +288,4 @@ public static TypeInformation[] transformTypes(Class[] fieldTypes){ return types; } -} \ No newline at end of file +} diff --git a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java deleted file mode 100644 index fde2f166e..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/util/JDBCUtils.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.util; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.DriverManager; - -public class JDBCUtils { - - private static final Logger LOG = LoggerFactory.getLogger(ClassUtil.class); - - public final static String lock_str = "jdbc_lock_str"; - - public static void forName(String clazz, ClassLoader classLoader) { - synchronized (lock_str){ - try { - Class.forName(clazz, true, classLoader); - DriverManager.setLoginTimeout(10); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - - - public synchronized static void forName(String clazz) { - try { - Class driverClass = Class.forName(clazz); - driverClass.newInstance(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} diff --git a/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java deleted file mode 100644 index 73f0e9baa..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/util/ParseUtils.java +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.util; - -import com.google.common.collect.Lists; -import org.apache.calcite.sql.*; -import org.apache.calcite.sql.fun.SqlCase; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.commons.lang3.StringUtils; -import java.util.List; -import java.util.Map; - -import static org.apache.calcite.sql.SqlKind.*; - -/** - * @Auther: jiangjunjie - * @Date: 2019-06-30 14:57 - * @Description: - */ -public class ParseUtils { - public static void parseSideWhere(SqlNode whereNode, Map physicalFields, List whereConditionList) { - SqlKind sqlKind = whereNode.getKind(); - if ((sqlKind == SqlKind.OR || sqlKind == SqlKind.AND) && ((SqlBasicCall) whereNode).getOperandList().size() == 2) { - SqlNode[] sqlOperandsList = ((SqlBasicCall) whereNode).getOperands(); - // whereNode是一颗先解析or再解析and的二叉树。二叉树中序遍历，先左子树，其次中间节点，最后右子树 - parseSideWhere(sqlOperandsList[0], physicalFields, whereConditionList); - whereConditionList.add(sqlKind.name()); - parseSideWhere(sqlOperandsList[1], physicalFields, whereConditionList); - } else { - SqlIdentifier sqlIdentifier = (SqlIdentifier) ((SqlBasicCall) whereNode).getOperands()[0]; - String fieldName = null; - if (sqlIdentifier.names.size() == 1) { - fieldName = sqlIdentifier.getComponent(0).getSimple(); - } else { - fieldName = sqlIdentifier.getComponent(1).getSimple(); - } - if (physicalFields.containsKey(fieldName)) { - String sideFieldName = physicalFields.get(fieldName); - // clone SqlIdentifier node - SqlParserPos sqlParserPos = new SqlParserPos(0, 0); - SqlIdentifier sqlIdentifierClone = new SqlIdentifier("", null, sqlParserPos); - List namesClone = Lists.newArrayList(); - for(String name :sqlIdentifier.names){ - namesClone.add(name); - } - sqlIdentifierClone.setNames(namesClone,null); - // clone SqlBasicCall node - SqlBasicCall sqlBasicCall = (SqlBasicCall)whereNode; - SqlNode[] sqlNodes = sqlBasicCall.getOperands(); - SqlNode[] sqlNodesClone = new SqlNode[sqlNodes.length]; - for (int i = 0; i < sqlNodes.length; i++) { - sqlNodesClone[i] = sqlNodes[i]; - } - SqlBasicCall sqlBasicCallClone = new SqlBasicCall(sqlBasicCall.getOperator(), sqlNodesClone, sqlParserPos); - // 替换维表中真实字段名 - List names = Lists.newArrayList(); - names.add(sideFieldName); - sqlIdentifierClone.setNames(names, null); - - sqlBasicCallClone.setOperand(0, sqlIdentifierClone); - whereConditionList.add(sqlBasicCallClone.toString()); - } else { - // 如果字段不是维表中字段，删除字段前的链接符 - if (whereConditionList.size() >= 1) { - whereConditionList.remove(whereConditionList.size() - 1); - } - } - } - } - - public static void parseAnd(SqlNode conditionNode, List sqlNodeList){ - if(conditionNode.getKind() == SqlKind.AND && ((SqlBasicCall)conditionNode).getOperandList().size()==2){ - parseAnd(((SqlBasicCall)conditionNode).getOperands()[0], sqlNodeList); - sqlNodeList.add(((SqlBasicCall)conditionNode).getOperands()[1]); - }else{ - sqlNodeList.add(conditionNode); - } - } - - public static void parseJoinCompareOperate(SqlNode condition, List sqlJoinCompareOperate) { - SqlBasicCall joinCondition = (SqlBasicCall) condition; - if (joinCondition.getKind() == SqlKind.AND) { - List operandList = joinCondition.getOperandList(); - for (SqlNode sqlNode : operandList) { - parseJoinCompareOperate(sqlNode, sqlJoinCompareOperate); - } - } else { - String operator = transformNotEqualsOperator(joinCondition.getKind()); - sqlJoinCompareOperate.add(operator); - } - } - - public static String transformNotEqualsOperator(SqlKind sqlKind) { - if (StringUtils.equalsIgnoreCase(sqlKind.toString(), "NOT_EQUALS")){ - return "!="; - } - return sqlKind.sql; - } - - public static SqlNode replaceJoinConditionTabName(SqlNode conditionNode, Map mappingTable) { - SqlNode[] operands = ((SqlBasicCall) conditionNode).getOperands(); - - for (int i = 0; i < operands.length; i++) { - SqlNode sqlNode = operands[i]; - SqlNode replaceNode = replaceNodeInfo(sqlNode, mappingTable); - operands[i] = replaceNode; - } - return conditionNode; - } - - /** - * m.id covert m_x_0.id - * @param selectNode - * @param mapTab - * @return - */ - public static SqlNode replaceSelectFieldTabName(SqlNode selectNode, Map mapTab) { - if (selectNode.getKind() == AS) { - SqlNode leftNode = ((SqlBasicCall) selectNode).getOperands()[0]; - SqlNode replaceNode = replaceSelectFieldTabName(leftNode, mapTab); - if (replaceNode != null) { - ((SqlBasicCall) selectNode).getOperands()[0] = replaceNode; - } - - return selectNode; - }else if(selectNode.getKind() == IDENTIFIER){ - SqlIdentifier sqlIdentifier = (SqlIdentifier) selectNode; - - if(sqlIdentifier.names.size() == 1){ - return selectNode; - } - - String newTableName = ParseUtils.getRootName(mapTab, sqlIdentifier.getComponent(0).getSimple()); - - if(newTableName == null){ - return selectNode; - } - sqlIdentifier = sqlIdentifier.setName(0, newTableName); - return sqlIdentifier; - - }else if(selectNode.getKind() == LITERAL || selectNode.getKind() == LITERAL_CHAIN){//字面含义 - return selectNode; - }else if( AGGREGATE.contains(selectNode.getKind()) - || AVG_AGG_FUNCTIONS.contains(selectNode.getKind()) - || COMPARISON.contains(selectNode.getKind()) - || selectNode.getKind() == OTHER_FUNCTION - || selectNode.getKind() == DIVIDE - || selectNode.getKind() == CAST - || selectNode.getKind() == TRIM - || selectNode.getKind() == TIMES - || selectNode.getKind() == PLUS - || selectNode.getKind() == NOT_IN - || selectNode.getKind() == OR - || selectNode.getKind() == AND - || selectNode.getKind() == MINUS - || selectNode.getKind() == TUMBLE - || selectNode.getKind() == TUMBLE_START - || selectNode.getKind() == TUMBLE_END - || selectNode.getKind() == SESSION - || selectNode.getKind() == SESSION_START - || selectNode.getKind() == SESSION_END - || selectNode.getKind() == HOP - || selectNode.getKind() == HOP_START - || selectNode.getKind() == HOP_END - || selectNode.getKind() == BETWEEN - || selectNode.getKind() == IS_NULL - || selectNode.getKind() == IS_NOT_NULL - || selectNode.getKind() == CONTAINS - || selectNode.getKind() == TIMESTAMP_ADD - || selectNode.getKind() == TIMESTAMP_DIFF - - ){ - SqlBasicCall sqlBasicCall = (SqlBasicCall) selectNode; - for(int i=0; i mapTab, SqlNodeList thenOperands) { - for(int i=0; i mapTab) { - if (parseNode.getKind() == IDENTIFIER) { - SqlIdentifier sqlIdentifier = (SqlIdentifier) parseNode; - - String newTableName = ParseUtils.getRootName(mapTab, sqlIdentifier.getComponent(0).getSimple());; - - if (newTableName == null || sqlIdentifier.names.size() == 1) { - return sqlIdentifier; - } - sqlIdentifier = sqlIdentifier.setName(0, newTableName); - return sqlIdentifier; - } else if (parseNode instanceof SqlBasicCall) { - SqlBasicCall sqlBasicCall = (SqlBasicCall) parseNode; - for (int i = 0; i < sqlBasicCall.getOperandList().size(); i++) { - SqlNode sqlNode = sqlBasicCall.getOperandList().get(i); - SqlNode replaceNode = replaceSelectFieldTabName(sqlNode, mapTab); - sqlBasicCall.getOperands()[i] = replaceNode; - } - - return sqlBasicCall; - } else { - return parseNode; - } - } - - - public static String getRootName(Map maps, String key) { - String res = null; - while (maps.get(key) !=null) { - res = maps.get(key); - key = res; - } - return res; - } - - public static void parseLeftNodeTableName(SqlNode leftJoin, List tablesName) { - if (leftJoin.getKind() == IDENTIFIER) { - SqlIdentifier sqlIdentifier = (SqlIdentifier) leftJoin; - tablesName.add(sqlIdentifier.names.get(0)); - } else if (leftJoin.getKind() == AS) { - SqlNode sqlNode = ((SqlBasicCall) leftJoin).getOperands()[1]; - tablesName.add(sqlNode.toString()); - } else if (leftJoin.getKind() == JOIN) { - parseLeftNodeTableName(((SqlJoin) leftJoin).getLeft(), tablesName); - parseLeftNodeTableName(((SqlJoin) leftJoin).getRight(), tablesName); - } - } -} diff --git a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java index f1c26b047..1f448b20a 100644 --- a/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java +++ b/core/src/main/java/com/dtstack/flink/sql/util/PluginUtil.java @@ -22,10 +22,10 @@ import com.dtstack.flink.sql.classloader.DtClassLoader; import org.apache.commons.lang3.StringUtils; -import com.fasterxml.jackson.core.JsonGenerationException; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerationException; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonParseException; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonMappingException; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import java.io.ByteArrayInputStream; import java.io.File; @@ -33,8 +33,6 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Properties; @@ -109,38 +107,20 @@ public static Properties stringToProperties(String str) throws IOException{ return properties; } - public static URL getRemoteJarFilePath(String pluginType, String tableType, String remoteSqlRootDir, String localSqlPluginPath) throws Exception { - return buildFinalJarFilePath(pluginType, tableType, remoteSqlRootDir, localSqlPluginPath); - } - - public static URL getLocalJarFilePath(String pluginType, String tableType, String localSqlPluginPath) throws Exception { - return buildFinalJarFilePath(pluginType, tableType, null, localSqlPluginPath); - } - - public static URL buildFinalJarFilePath(String pluginType, String tableType, String remoteSqlRootDir, String localSqlPluginPath) throws Exception { + public static URL getRemoteJarFilePath(String pluginType, String tableType, String remoteSqlRootDir) throws Exception { String dirName = pluginType + tableType.toLowerCase(); String prefix = String.format("%s-%s", pluginType, tableType.toLowerCase()); - String jarPath = localSqlPluginPath + SP + dirName; + String jarPath = remoteSqlRootDir + SP + dirName; String jarName = getCoreJarFileName(jarPath, prefix); - String sqlRootDir = remoteSqlRootDir == null ? localSqlPluginPath : remoteSqlRootDir; - return new URL("file:" + sqlRootDir + SP + dirName + SP + jarName); - } - - public static URL getRemoteSideJarFilePath(String pluginType, String sideOperator, String tableType, String remoteSqlRootDir, String localSqlPluginPath) throws Exception { - return buildFinalSideJarFilePath(pluginType, sideOperator, tableType, remoteSqlRootDir, localSqlPluginPath); + return new URL("file:" + remoteSqlRootDir + SP + dirName + SP + jarName); } - public static URL getLocalSideJarFilePath(String pluginType, String sideOperator, String tableType, String localSqlPluginPath) throws Exception { - return buildFinalSideJarFilePath(pluginType, sideOperator, tableType, null, localSqlPluginPath); - } - - public static URL buildFinalSideJarFilePath(String pluginType, String sideOperator, String tableType, String remoteSqlRootDir, String localSqlPluginPath) throws Exception { + public static URL getRemoteSideJarFilePath(String pluginType, String sideOperator, String tableType, String remoteSqlRootDir) throws Exception { String dirName = pluginType + sideOperator + tableType.toLowerCase(); String prefix = String.format("%s-%s-%s", pluginType, sideOperator, tableType.toLowerCase()); - String jarPath = localSqlPluginPath + SP + dirName; + String jarPath = remoteSqlRootDir + SP + dirName; String jarName = getCoreJarFileName(jarPath, prefix); - String sqlRootDir = remoteSqlRootDir == null ? localSqlPluginPath : remoteSqlRootDir; - return new URL("file:" + sqlRootDir + SP + dirName + SP + jarName); + return new URL("file:" + remoteSqlRootDir + SP + dirName + SP + jarName); } public static String upperCaseFirstChar(String str){ @@ -164,25 +144,6 @@ public static void addPluginJar(String pluginDir, DtClassLoader classLoader) thr } } - public static URL[] getPluginJarUrls(String pluginDir) throws MalformedURLException { - List urlList = new ArrayList<>(); - File dirFile = new File(pluginDir); - if(!dirFile.exists() || !dirFile.isDirectory()){ - throw new RuntimeException("plugin path:" + pluginDir + "is not exist."); - } - - File[] files = dirFile.listFiles(tmpFile -> tmpFile.isFile() && tmpFile.getName().endsWith(JAR_SUFFIX)); - if(files == null || files.length == 0){ - throw new RuntimeException("plugin path:" + pluginDir + " is null."); - } - - for(File file : files){ - URL pluginJarURL = file.toURI().toURL(); - urlList.add(pluginJarURL); - } - return urlList.toArray(new URL[urlList.size()]); - } - public static String getCoreJarFileName (String path, String prefix) throws Exception { String coreJarFileName = null; File pluginDir = new File(path); diff --git a/core/src/main/java/com/dtstack/flink/sql/util/PropertiesUtils.java b/core/src/main/java/com/dtstack/flink/sql/util/PropertiesUtils.java deleted file mode 100644 index dcb2a081a..000000000 --- a/core/src/main/java/com/dtstack/flink/sql/util/PropertiesUtils.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.util; - -import java.util.Properties; - -public class PropertiesUtils { - public static Properties propertiesTrim(Properties confProperties) { - Properties properties = new Properties(); - confProperties.forEach( - (k, v) -> { - properties.put(k.toString().trim(), v.toString().trim()); - } - ); - return properties; - } -} diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java index c9fbc0f44..b990bdd82 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/AbsCustomerWaterMarker.java @@ -1,22 +1,3 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - package com.dtstack.flink.sql.watermarker; import com.dtstack.flink.sql.metric.EventDelayGauge; diff --git a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java index d444c1bac..9288ecb87 100644 --- a/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java +++ b/core/src/main/java/com/dtstack/flink/sql/watermarker/WaterMarkerAssigner.java @@ -21,16 +21,16 @@ package com.dtstack.flink.sql.watermarker; import com.dtstack.flink.sql.table.SourceTableInfo; +import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.base.Strings; +import org.apache.flink.shaded.guava18.com.google.common.base.Strings; import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; -import java.util.TimeZone; - /** * define watermarker * Date: 2018/6/29 diff --git a/core/src/main/scala/com/dtstack/flink/App.scala b/core/src/main/scala/com/dtstack/flink/App.scala deleted file mode 100644 index e74ccc28a..000000000 --- a/core/src/main/scala/com/dtstack/flink/App.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.dtstack.flink - -/** - * Hello world! - * - */ -object App { - def main(args: Array[String]): Unit = { - println( "Hello World!" ) - } -} diff --git a/core/src/test/java/com/dtstack/flink/sql/side/SideSqlExecTest.java b/core/src/test/java/com/dtstack/flink/sql/side/SideSqlExecTest.java index 3bab778ae..8eb09ee18 100644 --- a/core/src/test/java/com/dtstack/flink/sql/side/SideSqlExecTest.java +++ b/core/src/test/java/com/dtstack/flink/sql/side/SideSqlExecTest.java @@ -22,10 +22,12 @@ import com.dtstack.flink.sql.Main; import com.dtstack.flink.sql.parser.SqlParser; import com.dtstack.flink.sql.parser.SqlTree; -import com.google.common.base.Charsets; -import com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.base.Charsets; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import org.junit.Test; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; import java.net.URLEncoder; import java.util.List; diff --git a/core/src/test/java/com/dtstack/flink/sql/side/TestSideSqlParser.java b/core/src/test/java/com/dtstack/flink/sql/side/TestSideSqlParser.java index 18a7d2d86..01e33a3b6 100644 --- a/core/src/test/java/com/dtstack/flink/sql/side/TestSideSqlParser.java +++ b/core/src/test/java/com/dtstack/flink/sql/side/TestSideSqlParser.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.side; import org.apache.calcite.sql.parser.SqlParseException; -import com.google.common.collect.Sets; +import org.apache.flink.calcite.shaded.com.google.common.collect.Sets; import org.junit.Test; import java.util.Set; diff --git a/docs/clickhouseSide.md b/docs/clickhouseSide.md deleted file mode 100644 index 63d3cc3da..000000000 --- a/docs/clickhouseSide.md +++ /dev/null @@ -1,85 +0,0 @@ - -## 1.格式： -``` - CREATE TABLE tableName( - colName cloType, - ... - PRIMARY KEY(keyInfo), - PERIOD FOR SYSTEM_TIME - )WITH( - type='clickhouse', - url='jdbcUrl', - userName='dbUserName', - password='dbPwd', - tableName='tableName', - cache ='LRU', - cacheSize ='10000', - cacheTTLMs ='60000', - parallelism ='1', - partitionedJoin='false' - ); -``` - -# 2.支持版本 - 19.14.x、19.15.x、19.16.x - -## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | clickhouse表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 clickhouse |是|| - | url | 连接clickhouse数据库 jdbcUrl |是|| - | userName | clickhouse连接用户名 |是|| - | password | clickhouse连接密码|是|| - | tableName | clickhouse表名称|是|| - | tableName | clickhouse 的表名称|是|| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - * cacheMode: (unordered|ordered)异步加载是有序还是无序,默认有序。 - * asyncCapacity:异步请求容量，默认1000 - * asyncTimeout：异步请求超时时间，默认10000毫秒 - -## 5.样例 -``` -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), - PERIOD FOR SYSTEM_TIME - )WITH( - type='clickhouse', - url='jdbc:clickhouse://172.16.8.104:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='sidetest', - cache ='LRU', - cacheSize ='10000', - cacheTTLMs ='60000', - cacheMode='unordered', - asyncCapacity='1000', - asyncTimeout='10000' - parallelism ='1', - partitionedJoin='false' - ); - - -``` - - diff --git a/docs/clickhouseSink.md b/docs/clickhouseSink.md deleted file mode 100644 index d9774727f..000000000 --- a/docs/clickhouseSink.md +++ /dev/null @@ -1,53 +0,0 @@ -## 1.格式： -``` -CREATE TABLE tableName( - colName colType, - ... - colNameX colType - )WITH( - type ='clickhouse', - url ='jdbcUrl', - userName ='userName', - password ='pwd', - tableName ='tableName', - parallelism ='parllNum' - ); - -``` - -## 2.支持版本 - 19.14.x、19.15.x、19.16.x - -## 3.表结构定义 - -|参数名称|含义| -|----|---| -| tableName| clickhouse表名称| -| colName | 列名称| -| colType | 列类型 [colType支持的类型](colType.md)| - -## 4.参数： - -|参数名称|含义|是否必填|默认值| -|----|----|----|----| -|type |表明 输出表类型 clickhouse |是|| -|url | 连接clickhouse 数据库 jdbcUrl |是|| -|userName | clickhouse 连接用户名 |是|| -| password | clickhouse 连接密码|是|| -| tableName | clickhouse 表名称|是|| -| parallelism | 并行度设置|否|1| - -## 5.样例： -``` -CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR - )WITH( - type ='clickhouse', - url ='jdbc:clickhouse://172.16.8.104:3306/test?charset=utf8', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - parallelism ='1' - ) - ``` \ No newline at end of file diff --git a/docs/consoleSink.md b/docs/consoleSink.md deleted file mode 100644 index 206d7faaa..000000000 --- a/docs/consoleSink.md +++ /dev/null @@ -1,50 +0,0 @@ -## 1.格式： -``` -CREATE TABLE tableName( - colName colType, - ... - colNameX colType - )WITH( - type ='console', - parallelism ='parllNum' - ); - -``` - -## 2.支持版本 -没有限制 - -## 3.表结构定义 - -|参数名称|含义| -|----|---| -| tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| -| colName | 列名称| -| colType | 列类型 [colType支持的类型](colType.md)| - -## 4.参数： - -|参数名称|含义|是否必填|默认值| -|----|----|----|----| -|type |表明 输出表类型[console]|是|| -| parallelism | 并行度设置|否|1| - -## 5.样例： -``` -CREATE TABLE MyResult( - name VARCHAR, - channel VARCHAR - )WITH( - type ='console', - parallelism ='1' - ) - ``` - - ## 6.输出结果: - ``` - +------+---------+ - | name | channel | - +------+---------+ - | aa | 02 | - +------+---------+ - ``` \ No newline at end of file diff --git a/docs/elasticsearchSink.md b/docs/elasticsearchSink.md index 69d69af75..9a406245a 100644 --- a/docs/elasticsearchSink.md +++ b/docs/elasticsearchSink.md @@ -27,7 +27,7 @@ CREATE TABLE tableName( ## 4.参数： |参数名称|含义|是否必填|默认值| |----|---|---|----| -|type|表明 输出表类型[mysq|hbase|elasticsearch]|是|| +|type|表明 输出表类型[mysql\|hbase\|elasticsearch]|是|| |address | 连接ES Transport地址(tcp地址)|是|| |cluster | ES 集群名称 |是|| |index | 选择的ES上的index名称|是|| diff --git a/docs/hbaseSide.md b/docs/hbaseSide.md index 0c4e545f9..07b03026a 100644 --- a/docs/hbaseSide.md +++ b/docs/hbaseSide.md @@ -35,7 +35,7 @@ |参数名称|含义|是否必填|默认值| |----|---|---|----| -| type | 表明维表的类型[hbase|mysql]|是|| +| type | 表明维表的类型[hbase\|mysql]|是|| | zookeeperQuorum | hbase 的zk地址;格式ip:port[;ip:port]|是|| | zookeeperParent | hbase 的zk parent路径|是|| | tableName | hbase 的表名称|是|| diff --git a/docs/hbaseSink.md b/docs/hbaseSink.md index c6a15e766..b41abb281 100644 --- a/docs/hbaseSink.md +++ b/docs/hbaseSink.md @@ -30,7 +30,7 @@ hbase2.0 |参数名称|含义|是否必填|默认值| |----|---|---|-----| -|type | 表明 输出表类型[mysq|hbase|elasticsearch]|是|| +|type | 表明 输出表类型[mysql\|hbase\|elasticsearch]|是|| |zookeeperQuorum | hbase zk地址,多个直接用逗号隔开|是|| |zookeeperParent | zkParent 路径|是|| |tableName | 关联的hbase表名称|是|| diff --git a/docs/kafkaSource.md b/docs/kafkaSource.md index 580eda6b4..748948a7b 100644 --- a/docs/kafkaSource.md +++ b/docs/kafkaSource.md @@ -9,20 +9,16 @@ CREATE TABLE tableName( WATERMARK FOR colName AS withOffset( colName , delayTime ) )WITH( type ='kafka09', - kafka.bootstrap.servers ='ip:port,ip:port...', - kafka.zookeeper.quorum ='ip:port,ip:port/zkparent', - kafka.auto.offset.reset ='latest', - kafka.topic ='topicName', - parallelism ='parllNum', - --timezone='America/Los_Angeles', - timezone='Asia/Shanghai', - sourcedatatype ='json' #可不设置 + bootstrapServers ='ip:port,ip:port...', + zookeeperQuorum ='ip:port,ip:port/zkparent', + offsetReset ='latest', + topic ='topicName', + parallelism ='parllNum' ); ``` ## 2.支持的版本 - kafka08,kafka09,kafka10,kafka11及以上版本 - **kafka读取和写入的版本必须一致，否则会有兼容性错误。** + kafka09,kafka10,kafka11 ## 3.表结构定义 @@ -32,24 +28,21 @@ CREATE TABLE tableName( | colName | 列名称| | colType | 列类型 [colType支持的类型](colType.md)| | function(colNameX) as aliasName | 支持在定义列信息的时候根据已有列类型生成新的列(函数可以使用系统函数和已经注册的UDF)| -| WATERMARK FOR colName AS withOffset( colName , delayTime ) | 标识输入流生的watermake生成规则,根据指定的colName(当前支持列的类型为Long | Timestamp) 和delayTime生成waterMark 同时会在注册表的使用附带上rowtime字段(如果未指定则默认添加proctime字段);注意：添加该标识的使用必须设置系统参数 time.characteristic:EventTime; delayTime: 数据最大延迟时间(ms)| +| WATERMARK FOR colName AS withOffset( colName , delayTime ) | 标识输入流生的watermake生成规则,根据指定的colName(当前支持列的类型为Long \| Timestamp) 和delayTime生成waterMark 同时会在注册表的使用附带上rowtime字段(如果未指定则默认添加proctime字段);注意：添加该标识的使用必须设置系统参数 time.characteristic:EventTime; delayTime: 数据最大延迟时间(ms)| ## 4.参数： |参数名称|含义|是否必填|默认值| |----|---|---|---| -|type | kafka09 | 是|kafka08、kafka09、kafka10、kafka11、kafka(对应kafka1.0及以上版本)| -|kafka.group.id | 需要读取的 groupId 名称|否|| -|kafka.bootstrap.servers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|kafka.zookeeper.quorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|kafka.topic | 需要读取的 topic 名称|是|| -|patterntopic | topic是否是正则表达式格式(true|false) |否| false -|kafka.auto.offset.reset | 读取的topic 的offset初始位置[latest|earliest|指定offset值({"0":12312,"1":12321,"2":12312},{"partition_no":offset_value})]|否|latest| +|type | kafka09 | 是|| +|bootstrapServers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| +|zookeeperQuorum | kafka zk地址信息(多个之间用逗号分隔)|是|| +|topic | 需要读取的 topic 名称|是|| +|topicIsPattern | topic是否是正则表达式格式|否| false +|groupId | 需要读取的 groupId 名称|否|| +|offsetReset | 读取的topic 的offset初始位置[latest\|earliest\|指定offset值({"0":12312,"1":12321,"2":12312},{"partition_no":offset_value})]|否|latest| |parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|json| -|timezone|时区设置[timezone支持的参数](timeZone.md)|否|'Asia/Shanghai' -**kafka相关参数可以自定义，使用kafka.开头即可。** - + ## 5.样例： ``` CREATE TABLE MyTable( @@ -58,75 +51,6 @@ CREATE TABLE MyTable( pv INT, xctime bigint, CHARACTER_LENGTH(channel) AS timeLeng - )WITH( - type ='kafka09', - kafka.bootstrap.servers ='172.16.8.198:9092', - kafka.zookeeper.quorum ='172.16.8.198:2181/kafka', - kafka.auto.offset.reset ='latest', - kafka.topic ='nbTest1,nbTest2,nbTest3', - --kafka.topic ='mqTest.*', - --patterntopic='true' - parallelism ='1', - sourcedatatype ='json' #可不设置 - ); -``` -## 6.支持嵌套json、数据类型字段解析 - -嵌套json解析示例 - -json: {"name":"tom", "obj":{"channel": "root"}, "pv": 4, "xctime":1572932485} -``` -CREATE TABLE MyTable( - name varchar, - obj.channel varchar as channel, - pv INT, - xctime bigint, - CHARACTER_LENGTH(channel) AS timeLeng - )WITH( - type ='kafka09', - bootstrapServers ='172.16.8.198:9092', - zookeeperQuorum ='172.16.8.198:2181/kafka', - offsetReset ='latest', - groupId='nbTest', - topic ='nbTest1,nbTest2,nbTest3', - --- topic ='mqTest.*', - ---topicIsPattern='true', - parallelism ='1' - ); -``` - -数组类型字段解析示例 - -json: {"name":"tom", "obj":{"channel": "root"}, "user": [{"pv": 4}, {"pv": 10}], "xctime":1572932485} -``` -CREATE TABLE MyTable( - name varchar, - obj.channel varchar as channel, - user[1].pv INT as pv, - xctime bigint, - CHARACTER_LENGTH(channel) AS timeLeng - )WITH( - type ='kafka09', - bootstrapServers ='172.16.8.198:9092', - zookeeperQuorum ='172.16.8.198:2181/kafka', - offsetReset ='latest', - groupId='nbTest', - topic ='nbTest1,nbTest2,nbTest3', - --- topic ='mqTest.*', - ---topicIsPattern='true', - parallelism ='1' - ); -``` -or - -json: {"name":"tom", "obj":{"channel": "root"}, "pv": [4, 7, 10], "xctime":1572932485} -``` -CREATE TABLE MyTable( - name varchar, - obj.channel varchar as channel, - pv[1] INT as pv, - xctime bigint, - CHARACTER_LENGTH(channel) AS timeLeng )WITH( type ='kafka09', bootstrapServers ='172.16.8.198:9092', @@ -139,220 +63,3 @@ CREATE TABLE MyTable( parallelism ='1' ); ``` -# 二、csv格式数据源 -根据字段分隔符进行数据分隔，按顺序匹配sql中配置的列。如数据分隔列数和sql中配置的列数相等直接匹配；如不同参照lengthcheckpolicy策略处理。 -## 1.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|| -|kafka.bootstrap.servers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|kafka.zookeeper.quorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|kafka.topic | 需要读取的 topic 名称|是|| -|kafka.auto.offset.reset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| -|parallelism | 并行度设置 |否|1| -|sourcedatatype | 数据类型|是 |csv| -|fielddelimiter | 字段分隔符|是 || -|lengthcheckpolicy | 单行字段条数检查策略 |否|可选，默认为SKIP,其它可选值为EXCEPTION、PAD。SKIP：字段数目不符合时跳过 。EXCEPTION:字段数目不符合时抛出异常。PAD:按顺序填充，不存在的置为null。| -**kafka相关参数可以自定义，使用kafka.开头即可。** - -## 2.样例： -``` -CREATE TABLE MyTable( - name varchar, - channel varchar, - pv INT, - xctime bigint, - CHARACTER_LENGTH(channel) AS timeLeng - )WITH( - type ='kafka09', - kafka.bootstrap.servers ='172.16.8.198:9092', - kafka.zookeeper.quorum ='172.16.8.198:2181/kafka', - kafka.auto.offset.reset ='latest', - kafka.topic ='nbTest1', - --kafka.topic ='mqTest.*', - --kafka.topicIsPattern='true' - parallelism ='1', - sourcedatatype ='csv', - fielddelimiter ='\|', - lengthcheckpolicy = 'PAD' - ); - ``` -# 三、text格式数据源UDF自定义拆分 -Kafka源表数据解析流程：Kafka Source Table -> UDTF ->Realtime Compute -> SINK。从Kakfa读入的数据，都是VARBINARY（二进制）格式，对读入的每条数据，都需要用UDTF将其解析成格式化数据。 - 与其他格式不同，本格式定义DDL必须与以下SQL一摸一样，表中的五个字段顺序务必保持一致： - -## 1. 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 -``` -create table kafka_stream( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, -) with ( - type ='kafka09', - kafka.bootstrap.servers ='172.16.8.198:9092', - kafka.zookeeper.quorum ='172.16.8.198:2181/kafka', - kafka.auto.offset.reset ='latest', - kafka.topic ='nbTest1', - parallelism ='1', - sourcedatatype='text' - ） -``` -## 2.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | kafka09 | 是|| -|kafka.bootstrap.servers | kafka bootstrap-server 地址信息(多个用逗号隔开)|是|| -|kafka.zookeeper.quorum | kafka zk地址信息(多个之间用逗号分隔)|是|| -|kafka.topic | 需要读取的 topic 名称|是|| -|kafka.auto.offset.reset | 读取的topic 的offset初始位置[latest|earliest]|否|latest| -|parallelism | 并行度设置|否|1| -|sourcedatatype | 数据类型|否|text| -**kafka相关参数可以自定义，使用kafka.开头即可。** - -## 2.自定义： -从kafka读出的数据，需要进行窗口计算。 按照实时计算目前的设计，滚窗/滑窗等窗口操作，需要（且必须）在源表DDL上定义Watermark。Kafka源表比较特殊。如果要以kafka中message字段中的的Event Time进行窗口操作， -需要先从message字段，使用UDX解析出event time，才能定义watermark。 在kafka源表场景中，需要使用计算列。 假设，kafka中写入的数据如下： -2018-11-11 00:00:00|1|Anna|female整个计算流程为：Kafka SOURCE->UDTF->Realtime Compute->RDS SINK（单一分隔符可直接使用类csv格式模板，自定义适用于更复杂的数据类型，本说明只做参考） - -**SQL** -``` --- 定义解析Kakfa message的UDTF - CREATE FUNCTION kafkapaser AS 'com.XXXX.kafkaUDTF'; - CREATE FUNCTION kafkaUDF AS 'com.XXXX.kafkaUDF'; - -- 定义源表，注意：kafka源表DDL字段必须与以下例子一模一样。WITH中参数可改。 - create table kafka_src ( - _topic STRING, - _messageKey STRING, - _message STRING, - _partition INT, - _offset BIGINT, - ctime AS TO_TIMESTAMP(kafkaUDF(_message)), -- 定义计算列，计算列可理解为占位符，源表中并没有这一列，其中的数据可经过下游计算得出。注意计算里的类型必须为timestamp才能在做watermark。 - watermark for ctime as withoffset(ctime,0) -- 在计算列上定义watermark - ) WITH ( - type = 'kafka010', -- Kafka Source类型，与Kafka版本强相关，目前支持的Kafka版本请参考本文档 - topic = 'test_kafka_topic', - ... - ); - create table rds_sink ( - name VARCHAR, - age INT, - grade VARCHAR, - updateTime TIMESTAMP - ) WITH( - type='mysql', - url='jdbc:mysql://localhost:3306/test', - tableName='test4', - userName='test', - password='XXXXXX' - ); - -- 使用UDTF，将二进制数据解析成格式化数据 - CREATE VIEW input_view ( - name, - age, - grade, - updateTime - ) AS - SELECT - COUNT(*) as cnt, - T.ctime, - T.order, - T.name, - T.sex - from - kafka_src as S, - LATERAL TABLE (kafkapaser _message)) as T ( - ctime, - order, - name, - sex - ) - Group BY T.sex, - TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 对input_view中输出的数据做计算 - CREATE VIEW view2 ( - cnt, - sex - ) AS - SELECT - COUNT(*) as cnt, - T.sex - from - input_view - Group BY sex, TUMBLE(ctime, INTERVAL '1' MINUTE); - -- 使用解析出的格式化数据进行计算，并将结果输出到RDS中 - insert into rds_sink - SELECT - cnt,sex - from view2; - ``` -**UDF&UDTF** -``` -package com.XXXX; - import com.XXXX.fastjson.JSONObject; - import org.apache.flink.table.functions.TableFunction; - import org.apache.flink.table.types.DataType; - import org.apache.flink.table.types.DataTypes; - import org.apache.flink.types.Row; - import java.io.UnsupportedEncodingException; - /** - 以下例子解析输入Kafka中的JSON字符串，并将其格式化输出 - **/ - public class kafkaUDTF extends TableFunction { - public void eval(byte[] message) { - try { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - // 提取JSON Object中各字段 - String ctime = Timestamp.valueOf(data.split('\\|')[0]); - String order = data.split('\\|')[1]; - String name = data.split('\\|')[2]; - String sex = data.split('\\|')[3]; - // 将解析出的字段放到要输出的Row()对象 - Row row = new Row(4); - row.setField(0, ctime); - row.setField(1, age); - row.setField(2, grade); - row.setField(3, updateTime); - System.out.println("Kafka message str ==>" + row.toString()); - // 输出一行 - collect(row); - } catch (ClassCastException e) { - System.out.println("Input data format error. Input data " + msg + "is not json string"); - } - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - } - @Override - // 如果返回值是Row，就必须重载实现这个方法，显式地告诉系统返回的字段类型 - // 定义输出Row()对象的字段类型 - public DataType getResultType(Object[] arguments, Class[] argTypes) { - return DataTypes.createRowType(DataTypes.TIMESTAMP,DataTypes.STRING, DataTypes.Integer, DataTypes.STRING,DataTypes.STRING); - } - } - - package com.dp58; - package com.dp58.sql.udx; - import org.apache.flink.table.functions.FunctionContext; - import org.apache.flink.table.functions.ScalarFunction; - public class KafkaUDF extends ScalarFunction { - // 可选，open方法可以不写 - // 需要import org.apache.flink.table.functions.FunctionContext; - public String eval(byte[] message) { - // 读入一个二进制数据，并将其转换为String格式 - String msg = new String(message, "UTF-8"); - return msg.split('\\|')[0]; - } - public long eval(String b, String c) { - return eval(b) + eval(c); - } - //可选，close方法可以不写 - @Override - public void close() { - } - } - ``` diff --git a/docs/kuduSide.md b/docs/kuduSide.md index 5a73596ad..fece4a480 100644 --- a/docs/kuduSide.md +++ b/docs/kuduSide.md @@ -64,21 +64,21 @@ kudu 1.9.0+cdh6.2.0 |参数名称|含义|是否必填|默认值| |----|---|---|-----| -|type | 表明维表的类型[hbase|mysql|kudu]|是|| +|type | 表明维表的类型[hbase\|mysql|\kudu]|是|| | kuduMasters | kudu master节点的地址;格式ip[ip，ip2]|是|| | tableName | kudu 的表名称|是|| -| workerCount | 工作线程数 |否|| -| defaultOperationTimeoutMs | 写入操作超时时间 |否|| -| defaultSocketReadTimeoutMs | socket读取超时时间 |否|| -| primaryKey | 需要过滤的主键 ALL模式独有 |否|| -| lowerBoundPrimaryKey | 需要过滤的主键的最小值 ALL模式独有 |否|| -| upperBoundPrimaryKey | 需要过滤的主键的最大值(不包含) ALL模式独有 |否|| -| workerCount | 工作线程数 |否|| -| defaultOperationTimeoutMs | 写入操作超时时间 |否|| -| defaultSocketReadTimeoutMs | socket读取超时时间 |否|| -| batchSizeBytes |返回数据的大小 | 否|| -| limitNum |返回数据的条数 | 否|| -| isFaultTolerant |查询是否容错 查询失败是否扫描第二个副本 默认false 容错 | 否|| +| workerCount | 工作线程数 |否| +| defaultOperationTimeoutMs | 写入操作超时时间 |否| +| defaultSocketReadTimeoutMs | socket读取超时时间 |否| +| primaryKey | 需要过滤的主键 ALL模式独有 |否| +| lowerBoundPrimaryKey | 需要过滤的主键的最小值 ALL模式独有 |否| +| upperBoundPrimaryKey | 需要过滤的主键的最大值(不包含) ALL模式独有 |否| +| workerCount | 工作线程数 |否| +| defaultOperationTimeoutMs | 写入操作超时时间 |否| +| defaultSocketReadTimeoutMs | socket读取超时时间 |否| +| batchSizeBytes |返回数据的大小 | 否| +| limitNum |返回数据的条数 | 否| +| isFaultTolerant |查询是否容错 查询失败是否扫描第二个副本 默认false 容错 | 否| | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| diff --git a/docs/kuduSink.md b/docs/kuduSink.md index 990dfdd7d..ba607c7ec 100644 --- a/docs/kuduSink.md +++ b/docs/kuduSink.md @@ -34,10 +34,10 @@ kudu 1.9.0+cdh6.2.0 |参数名称|含义|是否必填|默认值| |----|---|---|-----| -|type | 表名 输出表类型[mysq|hbase|elasticsearch|redis|kudu]|是|| +|type | 表明 输出表类型[mysql\|hbase\|elasticsearch\redis\|kudu\]|是|| | kuduMasters | kudu master节点的地址;格式ip[ip，ip2]|是|| | tableName | kudu 的表名称|是|| -| writeMode | 写入kudu的模式 insert|update|upsert |否 |upsert +| writeMode | 写入kudu的模式 insert|update|upsert |否 |upsert | workerCount | 工作线程数 |否| | defaultOperationTimeoutMs | 写入操作超时时间 |否| | defaultSocketReadTimeoutMs | socket读取超时时间 |否| diff --git a/docs/mysqlSide.md b/docs/mysqlSide.md index f0eb16090..d0fec5832 100644 --- a/docs/mysqlSide.md +++ b/docs/mysqlSide.md @@ -27,7 +27,7 @@ |参数名称|含义| |----|---| - | tableName | mysql表名称| + | tableName | 注册到flink的表名称(可选填;不填默认和hbase对应的表名称相同)| | colName | 列名称| | colType | 列类型 [colType支持的类型](colType.md)| | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| @@ -37,7 +37,7 @@ |参数名称|含义|是否必填|默认值| |----|---|---|----| - | type | 表明维表的类型 mysql |是|| + | type | 表明维表的类型[hbase\|mysql] |是|| | url | 连接mysql数据库 jdbcUrl |是|| | userName | mysql连接用户名 |是|| | password | mysql连接密码|是|| @@ -52,9 +52,7 @@ * LRU: * cacheSize: 缓存的条目数量 * cacheTTLMs:缓存的过期时间(ms) - * cacheMode: (unordered|ordered)异步加载是有序还是无序,默认有序。 - * asyncCapacity:异步请求容量，默认1000 - * asyncTimeout：异步请求超时时间，默认10000毫秒 + ## 5.样例 ``` @@ -72,9 +70,6 @@ create table sideTable( cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000', - cacheMode='unordered', - asyncCapacity='1000', - asyncTimeout='10000' parallelism ='1', partitionedJoin='false' ); diff --git a/docs/mysqlSink.md b/docs/mysqlSink.md index 3218f9371..192c4944a 100644 --- a/docs/mysqlSink.md +++ b/docs/mysqlSink.md @@ -22,7 +22,7 @@ CREATE TABLE tableName( |参数名称|含义| |----|---| -| tableName| mysql表名称| +| tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| | colName | 列名称| | colType | 列类型 [colType支持的类型](colType.md)| @@ -30,7 +30,7 @@ CREATE TABLE tableName( |参数名称|含义|是否必填|默认值| |----|----|----|----| -|type |表名 输出表类型[mysq|hbase|elasticsearch]|是|| +|type |表明 输出表类型[mysql\|hbase\|elasticsearch]|是|| |url | 连接mysql数据库 jdbcUrl |是|| |userName | mysql连接用户名 |是|| | password | mysql连接密码|是|| diff --git a/docs/oracleSide.md b/docs/oracleSide.md deleted file mode 100644 index 74fc56680..000000000 --- a/docs/oracleSide.md +++ /dev/null @@ -1,85 +0,0 @@ - -## 1.格式： -``` - CREATE TABLE tableName( - colName cloType, - ... - PRIMARY KEY(keyInfo), - PERIOD FOR SYSTEM_TIME - )WITH( - type='oracle', - url='jdbcUrl', - userName='dbUserName', - password='dbPwd', - tableName='tableName', - cache ='LRU', - schema = 'MQTEST', - parallelism ='1', - partitionedJoin='false' - ); -``` - -# 2.支持版本 - 10g 11g - -## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | oracle表名称| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型 oracle |是|| - | url | 连接oracle数据库 jdbcUrl |是|| - | userName | oracle连接用户名 |是|| - | password | oracle连接密码|是|| - | tableName | oracle表名称|是|| - | schema | oracle 的schema|否|当前登录用户| - | cache | 维表缓存策略(NONE/LRU)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - * cacheMode: (unordered|ordered)异步加载是有序还是无序,默认有序。 - * asyncCapacity:异步请求容量，默认1000 - * asyncTimeout：异步请求超时时间，默认10000毫秒 - -## 5.样例 -``` -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), - PERIOD FOR SYSTEM_TIME - )WITH( - type='oracle', - url='jdbc:oracle:thin:@xx.xx.xx.xx:1521:orcl', - userName='xx', - password='xx', - tableName='sidetest', - cache ='LRU', - cacheSize ='10000', - cacheTTLMs ='60000', - cacheMode='unordered', - asyncCapacity='1000', - asyncTimeout='10000' - parallelism ='1', - partitionedJoin='false', - schema = 'MQTEST' - ); - - -``` - - diff --git a/docs/oracleSink.md b/docs/oracleSink.md deleted file mode 100644 index 47ddd8371..000000000 --- a/docs/oracleSink.md +++ /dev/null @@ -1,55 +0,0 @@ -## 1.格式： -``` -CREATE TABLE tableName( - colName colType, - ... - colNameX colType - )WITH( - type ='oracle', - url ='jdbcUrl', - userName ='userName', - password ='pwd', - tableName ='tableName', - parallelism ='parllNum' - ); - -``` - -## 2.支持版本 - 10g 11g - -## 3.表结构定义 - -|参数名称|含义| -|----|---| -| tableName| oracle表名称| -| colName | 列名称| -| colType | 列类型 [colType支持的类型](colType.md)| - -## 4.参数： - -|参数名称|含义|是否必填|默认值| -|----|----|----|----| -|type |表名 输出表类型[mysq|hbase|elasticsearch|oracle]|是|| -|url | 连接oracle数据库 jdbcUrl |是|| -|userName | oracle连接用户名 |是|| -| password | oracle连接密码|是|| -| tableName | oracle表名称|是|| -| schema | oracle 的schema|否|当前登录用户| -| parallelism | 并行度设置|否|1| - -## 5.样例： -``` -CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR - )WITH( - type ='oracle', - url ='jdbc:oracle:thin:@xx.xx.xx.xx:1521:orcl', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - schema = 'MQTEST', - parallelism ='1' - ) - ``` \ No newline at end of file diff --git a/docs/postgresqlSide.md b/docs/postgresqlSide.md deleted file mode 100644 index 68d10b869..000000000 --- a/docs/postgresqlSide.md +++ /dev/null @@ -1,80 +0,0 @@ - -## 1.格式： -``` - CREATE TABLE tableName( - colName cloType, - ... - PRIMARY KEY(keyInfo), - PERIOD FOR SYSTEM_TIME - )WITH( - type='postgresql', - url='jdbcUrl', - userName='dbUserName', - password='dbPwd', - tableName='tableName', - cache ='LRU', - cacheSize ='10000', - cacheTTLMs ='60000', - parallelism ='1', - partitionedJoin='false' - ); -``` - -# 2.支持版本 - postgresql-8.2+ - -## 3.表结构定义 - - |参数名称|含义| - |----|---| - | tableName | 注册到flink的表名称(可选填;不填默认和hbase对应的表名称相同)| - | colName | 列名称| - | colType | 列类型 [colType支持的类型](colType.md)| - | PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息| - | PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开| - -## 4.参数 - - |参数名称|含义|是否必填|默认值| - |----|---|---|----| - | type | 表明维表的类型[postgresql] |是|| - | url | 连接postgresql数据库 jdbcUrl |是|| - | userName | postgresql连接用户名 |是|| - | password | postgresql连接密码|是|| - | tableName | postgresql表名称|是|| - | tableName | postgresql 的表名称|是|| - | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| - | partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false| - - ---------- - > 缓存策略 - * NONE: 不做内存缓存 - * LRU: - * cacheSize: 缓存的条目数量 - * cacheTTLMs:缓存的过期时间(ms) - - -## 5.样例 -``` -create table sideTable( - channel varchar, - xccount int, - PRIMARY KEY(channel), - PERIOD FOR SYSTEM_TIME - )WITH( - type='postgresql', - url='jdbc:postgresql://localhost:9001/test?sslmode=disable', - userName='dtstack', - password='abc123', - tableName='sidetest', - cache ='LRU', - cacheSize ='10000', - cacheTTLMs ='60000', - parallelism ='1', - partitionedJoin='false' - ); - - -``` - - diff --git a/docs/postgresqlSink.md b/docs/postgresqlSink.md deleted file mode 100644 index da09c4c34..000000000 --- a/docs/postgresqlSink.md +++ /dev/null @@ -1,55 +0,0 @@ -## 1.格式： -``` -CREATE TABLE tableName( - colName colType, - ... - colNameX colType - )WITH( - type ='postgresql', - url ='jdbcUrl', - userName ='userName', - password ='pwd', - tableName ='tableName', - parallelism ='parllNum' - ); - -``` - -## 2.支持版本 - postgresql-8.2+ - -## 3.表结构定义 - -|参数名称|含义| -|----|---| -| tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称| -| colName | 列名称| -| colType | 列类型 [colType支持的类型](colType.md)| - -## 4.参数： - -|参数名称|含义|是否必填|默认值| -|----|----|----|----| -| type |表明 输出表类型[postgresql]|是|| -| url | 连接postgresql数据库 jdbcUrl |是|| -| userName | postgresql连接用户名 |是|| -| password | postgresql连接密码|是|| -| tableName | postgresqll表名称|是|| -| parallelism | 并行度设置|否|1| -| isUpsert | 使用upsert模式插入数据(版本9.5之后才支持upsert) |否|false -| keyField | 设置更新主键字段名（isupsert为true时为必填项）|否| - -## 5.样例： -``` -CREATE TABLE MyResult( - channel VARCHAR, - pv VARCHAR - )WITH( - type ='postgresql', - url ='jdbc:postgresql://localhost:9001/test?sslmode=disable', - userName ='dtstack', - password ='abc123', - tableName ='pv2', - parallelism ='1' - ) - ``` diff --git a/docs/redisSide.md b/docs/redisSide.md index 9c7f4b47e..52d637b70 100644 --- a/docs/redisSide.md +++ b/docs/redisSide.md @@ -11,7 +11,6 @@ password = 'redisPwd', database = 'dbName', tableName ='sideTableName', - redisType = '1', cache ='LRU', cacheSize ='10000', cacheTTLMs ='60000' @@ -33,11 +32,9 @@ |参数名称|含义|是否必填|默认值| |----|---|---|----| -| type | 表明维表的类型[hbase|mysql|redis]|是|| +| type | 表明维表的类型[hbase\|mysql\|redis]|是|| | url | redis 的地址;格式ip:port[,ip:port]|是|| | password | redis 的密码 |是|| -| redisType | redis模式（1 单机，2 哨兵， 3 集群）| 是 | -| masterName | 主节点名称（哨兵模式下为必填项） | 否 | | database | reids 的数据库地址|否|| | tableName | redis 的表名称|是|| | cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE| @@ -63,7 +60,6 @@ create table sideTable( url='172.16.10.79:6379', password='abc123', database='0', - redisType = '1', tableName='sidetest', cache = 'LRU', cacheTTLMs='10000' diff --git a/docs/redisSink.md b/docs/redisSink.md index 6a754e5c6..a3c80914e 100644 --- a/docs/redisSink.md +++ b/docs/redisSink.md @@ -9,7 +9,6 @@ CREATE TABLE tableName( url = 'ip:port', database ='dbName', password ='pwd', - redisType='1', tableName ='tableName', parallelism ='parllNum' ); @@ -33,14 +32,12 @@ redis5.0 |参数名称|含义|是否必填|默认值| |----|---|---|-----| -| type | 表名 输出表类型[mysq|hbase|elasticsearch|redis]|是|| +|type | 表明 输出表类型[mysql\|hbase\|elasticsearch\|redis\]|是|| | url | redis 的地址;格式ip:port[,ip:port]|是|| | password | redis 的密码 |是|| -| redisType | redis模式（1 单机，2 哨兵， 3 集群）| 是 | -| masterName | 主节点名称（哨兵模式下为必填项） | 否 | | database | reids 的数据库地址|否|| | tableName | redis 的表名称|是|| -| parallelism | 并行度设置|否|1| +|parallelism | 并行度设置|否|1| ## 5.样例： @@ -54,8 +51,7 @@ redis5.0 url='172.16.10.79:6379', password='abc123', database='0', - redisType='1', - tableName='sinktoredis' + tableName='sinktoredis', ); ``` \ No newline at end of file diff --git a/docs/serverSocketSource.md b/docs/serverSocketSource.md deleted file mode 100644 index 4f889d829..000000000 --- a/docs/serverSocketSource.md +++ /dev/null @@ -1,37 +0,0 @@ - -## 1.数据格式： -``` -数据现在只支持json格式 {"xx":"bb","cc":"dd"} - -CREATE TABLE MyTable( - channel varchar, - pv int, - xctime date, - xtime date - - )WITH( - type='serversocket', - host='127.0.0.1', - port='8888', - delimiter=';', - maxNumRetries='100' - ); -``` - - -## 2.参数： - -|参数名称|含义|是否必填|默认值| -|----|---|---|---| -|type | serversocket | 是|| -|host | server host|是|| -|port | server port|是|| -|delimiter| 每条json数据的分割符(比如：;)|是|| -|maxNumRetries| 最大重连次数 (大于0)|是|| - - -## 3.Server端样例： -``` -String JsonStr = "{\"CHANNEL\":\"xc3\",\"pv\":1234567,\"xdate\":\"2018-12-07\",\"xtime\":\"2018-12-15\"};"; - -``` diff --git a/docs/timeZone.md b/docs/timeZone.md deleted file mode 100644 index ec6aa0ca3..000000000 --- a/docs/timeZone.md +++ /dev/null @@ -1,601 +0,0 @@ -* Africa/Abidjan -* Africa/Accra -* Africa/Addis_Ababa -* Africa/Algiers -* Africa/Asmara -* Africa/Asmera -* Africa/Bamako -* Africa/Bangui -* Africa/Banjul -* Africa/Bissau -* Africa/Blantyre -* Africa/Brazzaville -* Africa/Bujumbura -* Africa/Cairo -* Africa/Casablanca -* Africa/Ceuta -* Africa/Conakry -* Africa/Dakar -* Africa/Dar_es_Salaam -* Africa/Djibouti -* Africa/Douala -* Africa/El_Aaiun -* Africa/Freetown -* Africa/Gaborone -* Africa/Harare -* Africa/Johannesburg -* Africa/Juba -* Africa/Kampala -* Africa/Khartoum -* Africa/Kigali -* Africa/Kinshasa -* Africa/Lagos -* Africa/Libreville -* Africa/Lome -* Africa/Luanda -* Africa/Lubumbashi -* Africa/Lusaka -* Africa/Malabo -* Africa/Maputo -* Africa/Maseru -* Africa/Mbabane -* Africa/Mogadishu -* Africa/Monrovia -* Africa/Nairobi -* Africa/Ndjamena -* Africa/Niamey -* Africa/Nouakchott -* Africa/Ouagadougou -* Africa/Porto-Novo -* Africa/Sao_Tome -* Africa/Timbuktu -* Africa/Tripoli -* Africa/Tunis -* Africa/Windhoek -* America/Adak -* America/Anchorage -* America/Anguilla -* America/Antigua -* America/Araguaina -* America/Argentina/Buenos_Aires -* America/Argentina/Catamarca -* America/Argentina/ComodRivadavia -* America/Argentina/Cordoba -* America/Argentina/Jujuy -* America/Argentina/La_Rioja -* America/Argentina/Mendoza -* America/Argentina/Rio_Gallegos -* America/Argentina/Salta -* America/Argentina/San_Juan -* America/Argentina/San_Luis -* America/Argentina/Tucuman -* America/Argentina/Ushuaia -* America/Aruba -* America/Asuncion -* America/Atikokan -* America/Atka -* America/Bahia -* America/Bahia_Banderas -* America/Barbados -* America/Belem -* America/Belize -* America/Blanc-Sablon -* America/Boa_Vista -* America/Bogota -* America/Boise -* America/Buenos_Aires -* America/Cambridge_Bay -* America/Campo_Grande -* America/Cancun -* America/Caracas -* America/Catamarca -* America/Cayenne -* America/Cayman -* America/Chicago -* America/Chihuahua -* America/Coral_Harbour -* America/Cordoba -* America/Costa_Rica -* America/Creston -* America/Cuiaba -* America/Curacao -* America/Danmarkshavn -* America/Dawson -* America/Dawson_Creek -* America/Denver -* America/Detroit -* America/Dominica -* America/Edmonton -* America/Eirunepe -* America/El_Salvador -* America/Ensenada -* America/Fort_Nelson -* America/Fort_Wayne -* America/Fortaleza -* America/Glace_Bay -* America/Godthab -* America/Goose_Bay -* America/Grand_Turk -* America/Grenada -* America/Guadeloupe -* America/Guatemala -* America/Guayaquil -* America/Guyana -* America/Halifax -* America/Havana -* America/Hermosillo -* America/Indiana/Indianapolis -* America/Indiana/Knox -* America/Indiana/Marengo -* America/Indiana/Petersburg -* America/Indiana/Tell_City -* America/Indiana/Vevay -* America/Indiana/Vincennes -* America/Indiana/Winamac -* America/Indianapolis -* America/Inuvik -* America/Iqaluit -* America/Jamaica -* America/Jujuy -* America/Juneau -* America/Kentucky/Louisville -* America/Kentucky/Monticello -* America/Knox_IN -* America/Kralendijk -* America/La_Paz -* America/Lima -* America/Los_Angeles -* America/Louisville -* America/Lower_Princes -* America/Maceio -* America/Managua -* America/Manaus -* America/Marigot -* America/Martinique -* America/Matamoros -* America/Mazatlan -* America/Mendoza -* America/Menominee -* America/Merida -* America/Metlakatla -* America/Mexico_City -* America/Miquelon -* America/Moncton -* America/Monterrey -* America/Montevideo -* America/Montreal -* America/Montserrat -* America/Nassau -* America/New_York -* America/Nipigon -* America/Nome -* America/Noronha -* America/North_Dakota/Beulah -* America/North_Dakota/Center -* America/North_Dakota/New_Salem -* America/Ojinaga -* America/Panama -* America/Pangnirtung -* America/Paramaribo -* America/Phoenix -* America/Port-au-Prince -* America/Port_of_Spain -* America/Porto_Acre -* America/Porto_Velho -* America/Puerto_Rico -* America/Punta_Arenas -* America/Rainy_River -* America/Rankin_Inlet -* America/Recife -* America/Regina -* America/Resolute -* America/Rio_Branco -* America/Rosario -* America/Santa_Isabel -* America/Santarem -* America/Santiago -* America/Santo_Domingo -* America/Sao_Paulo -* America/Scoresbysund -* America/Shiprock -* America/Sitka -* America/St_Barthelemy -* America/St_Johns -* America/St_Kitts -* America/St_Lucia -* America/St_Thomas -* America/St_Vincent -* America/Swift_Current -* America/Tegucigalpa -* America/Thule -* America/Thunder_Bay -* America/Tijuana -* America/Toronto -* America/Tortola -* America/Vancouver -* America/Virgin -* America/Whitehorse -* America/Winnipeg -* America/Yakutat -* America/Yellowknife -* Antarctica/Casey -* Antarctica/Davis -* Antarctica/DumontDUrville -* Antarctica/Macquarie -* Antarctica/Mawson -* Antarctica/McMurdo -* Antarctica/Palmer -* Antarctica/Rothera -* Antarctica/South_Pole -* Antarctica/Syowa -* Antarctica/Troll -* Antarctica/Vostok -* Arctic/Longyearbyen -* Asia/Aden -* Asia/Almaty -* Asia/Amman -* Asia/Anadyr -* Asia/Aqtau -* Asia/Aqtobe -* Asia/Ashgabat -* Asia/Ashkhabad -* Asia/Atyrau -* Asia/Baghdad -* Asia/Bahrain -* Asia/Baku -* Asia/Bangkok -* Asia/Barnaul -* Asia/Beirut -* Asia/Bishkek -* Asia/Brunei -* Asia/Calcutta -* Asia/Chita -* Asia/Choibalsan -* Asia/Chongqing -* Asia/Chungking -* Asia/Colombo -* Asia/Dacca -* Asia/Damascus -* Asia/Dhaka -* Asia/Dili -* Asia/Dubai -* Asia/Dushanbe -* Asia/Famagusta -* Asia/Gaza -* Asia/Harbin -* Asia/Hebron -* Asia/Ho_Chi_Minh -* Asia/Hong_Kong -* Asia/Hovd -* Asia/Irkutsk -* Asia/Istanbul -* Asia/Jakarta -* Asia/Jayapura -* Asia/Jerusalem -* Asia/Kabul -* Asia/Kamchatka -* Asia/Karachi -* Asia/Kashgar -* Asia/Kathmandu -* Asia/Katmandu -* Asia/Khandyga -* Asia/Kolkata -* Asia/Krasnoyarsk -* Asia/Kuala_Lumpur -* Asia/Kuching -* Asia/Kuwait -* Asia/Macao -* Asia/Macau -* Asia/Magadan -* Asia/Makassar -* Asia/Manila -* Asia/Muscat -* Asia/Nicosia -* Asia/Novokuznetsk -* Asia/Novosibirsk -* Asia/Omsk -* Asia/Oral -* Asia/Phnom_Penh -* Asia/Pontianak -* Asia/Pyongyang -* Asia/Qatar -* Asia/Qyzylorda -* Asia/Rangoon -* Asia/Riyadh -* Asia/Saigon -* Asia/Sakhalin -* Asia/Samarkand -* Asia/Seoul -* Asia/Shanghai -* Asia/Singapore -* Asia/Srednekolymsk -* Asia/Taipei -* Asia/Tashkent -* Asia/Tbilisi -* Asia/Tehran -* Asia/Tel_Aviv -* Asia/Thimbu -* Asia/Thimphu -* Asia/Tokyo -* Asia/Tomsk -* Asia/Ujung_Pandang -* Asia/Ulaanbaatar -* Asia/Ulan_Bator -* Asia/Urumqi -* Asia/Ust-Nera -* Asia/Vientiane -* Asia/Vladivostok -* Asia/Yakutsk -* Asia/Yangon -* Asia/Yekaterinburg -* Asia/Yerevan -* Atlantic/Azores -* Atlantic/Bermuda -* Atlantic/Canary -* Atlantic/Cape_Verde -* Atlantic/Faeroe -* Atlantic/Faroe -* Atlantic/Jan_Mayen -* Atlantic/Madeira -* Atlantic/Reykjavik -* Atlantic/South_Georgia -* Atlantic/St_Helena -* Atlantic/Stanley -* Australia/ACT -* Australia/Adelaide -* Australia/Brisbane -* Australia/Broken_Hill -* Australia/Canberra -* Australia/Currie -* Australia/Darwin -* Australia/Eucla -* Australia/Hobart -* Australia/LHI -* Australia/Lindeman -* Australia/Lord_Howe -* Australia/Melbourne -* Australia/NSW -* Australia/North -* Australia/Perth -* Australia/Queensland -* Australia/South -* Australia/Sydney -* Australia/Tasmania -* Australia/Victoria -* Australia/West -* Australia/Yancowinna -* Brazil/Acre -* Brazil/DeNoronha -* Brazil/East -* Brazil/West -* CET -* CST6CDT -* Canada/Atlantic -* Canada/Central -* Canada/Eastern -* Canada/Mountain -* Canada/Newfoundland -* Canada/Pacific -* Canada/Saskatchewan -* Canada/Yukon -* Chile/Continental -* Chile/EasterIsland -* Cuba -* EET -* EST5EDT -* Egypt -* Eire -* Etc/GMT -* Etc/GMT+0 -* Etc/GMT+1 -* Etc/GMT+10 -* Etc/GMT+11 -* Etc/GMT+12 -* Etc/GMT+2 -* Etc/GMT+3 -* Etc/GMT+4 -* Etc/GMT+5 -* Etc/GMT+6 -* Etc/GMT+7 -* Etc/GMT+8 -* Etc/GMT+9 -* Etc/GMT-0 -* Etc/GMT-1 -* Etc/GMT-10 -* Etc/GMT-11 -* Etc/GMT-12 -* Etc/GMT-13 -* Etc/GMT-14 -* Etc/GMT-2 -* Etc/GMT-3 -* Etc/GMT-4 -* Etc/GMT-5 -* Etc/GMT-6 -* Etc/GMT-7 -* Etc/GMT-8 -* Etc/GMT-9 -* Etc/GMT0 -* Etc/Greenwich -* Etc/UCT -* Etc/UTC -* Etc/Universal -* Etc/Zulu -* Europe/Amsterdam -* Europe/Andorra -* Europe/Astrakhan -* Europe/Athens -* Europe/Belfast -* Europe/Belgrade -* Europe/Berlin -* Europe/Bratislava -* Europe/Brussels -* Europe/Bucharest -* Europe/Budapest -* Europe/Busingen -* Europe/Chisinau -* Europe/Copenhagen -* Europe/Dublin -* Europe/Gibraltar -* Europe/Guernsey -* Europe/Helsinki -* Europe/Isle_of_Man -* Europe/Istanbul -* Europe/Jersey -* Europe/Kaliningrad -* Europe/Kiev -* Europe/Kirov -* Europe/Lisbon -* Europe/Ljubljana -* Europe/London -* Europe/Luxembourg -* Europe/Madrid -* Europe/Malta -* Europe/Mariehamn -* Europe/Minsk -* Europe/Monaco -* Europe/Moscow -* Europe/Nicosia -* Europe/Oslo -* Europe/Paris -* Europe/Podgorica -* Europe/Prague -* Europe/Riga -* Europe/Rome -* Europe/Samara -* Europe/San_Marino -* Europe/Sarajevo -* Europe/Saratov -* Europe/Simferopol -* Europe/Skopje -* Europe/Sofia -* Europe/Stockholm -* Europe/Tallinn -* Europe/Tirane -* Europe/Tiraspol -* Europe/Ulyanovsk -* Europe/Uzhgorod -* Europe/Vaduz -* Europe/Vatican -* Europe/Vienna -* Europe/Vilnius -* Europe/Volgograd -* Europe/Warsaw -* Europe/Zagreb -* Europe/Zaporozhye -* Europe/Zurich -* GB -* GB-Eire -* GMT -* GMT0 -* Greenwich -* Hongkong -* Iceland -* Indian/Antananarivo -* Indian/Chagos -* Indian/Christmas -* Indian/Cocos -* Indian/Comoro -* Indian/Kerguelen -* Indian/Mahe -* Indian/Maldives -* Indian/Mauritius -* Indian/Mayotte -* Indian/Reunion -* Iran -* Israel -* Jamaica -* Japan -* Kwajalein -* Libya -* MET -* MST7MDT -* Mexico/BajaNorte -* Mexico/BajaSur -* Mexico/General -* NZ -* NZ-CHAT -* Navajo -* PRC -* PST8PDT -* Pacific/Apia -* Pacific/Auckland -* Pacific/Bougainville -* Pacific/Chatham -* Pacific/Chuuk -* Pacific/Easter -* Pacific/Efate -* Pacific/Enderbury -* Pacific/Fakaofo -* Pacific/Fiji -* Pacific/Funafuti -* Pacific/Galapagos -* Pacific/Gambier -* Pacific/Guadalcanal -* Pacific/Guam -* Pacific/Honolulu -* Pacific/Johnston -* Pacific/Kiritimati -* Pacific/Kosrae -* Pacific/Kwajalein -* Pacific/Majuro -* Pacific/Marquesas -* Pacific/Midway -* Pacific/Nauru -* Pacific/Niue -* Pacific/Norfolk -* Pacific/Noumea -* Pacific/Pago_Pago -* Pacific/Palau -* Pacific/Pitcairn -* Pacific/Pohnpei -* Pacific/Ponape -* Pacific/Port_Moresby -* Pacific/Rarotonga -* Pacific/Saipan -* Pacific/Samoa -* Pacific/Tahiti -* Pacific/Tarawa -* Pacific/Tongatapu -* Pacific/Truk -* Pacific/Wake -* Pacific/Wallis -* Pacific/Yap -* Poland -* Portugal -* ROK -* Singapore -* SystemV/AST4 -* SystemV/AST4ADT -* SystemV/CST6 -* SystemV/CST6CDT -* SystemV/EST5 -* SystemV/EST5EDT -* SystemV/HST10 -* SystemV/MST7 -* SystemV/MST7MDT -* SystemV/PST8 -* SystemV/PST8PDT -* SystemV/YST9 -* SystemV/YST9YDT -* Turkey -* UCT -* US/Alaska -* US/Aleutian -* US/Arizona -* US/Central -* US/East-Indiana -* US/Eastern -* US/Hawaii -* US/Indiana-Starke -* US/Michigan -* US/Mountain -* US/Pacific -* US/Pacific-New -* US/Samoa -* UTC -* Universal -* W-SU -* WET -* Zulu - - diff --git a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java index e616163ec..19f158c3a 100644 --- a/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java +++ b/elasticsearch5/elasticsearch5-sink/src/main/java/com/dtstack/flink/sql/sink/elasticsearch/table/ElasticsearchTableInfo.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.table.TargetTableInfo; -import com.google.common.base.Preconditions; +import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; /** * @date 2018/09/12 diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java index 0fe2d1720..c9adc87b9 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllReqRow.java @@ -24,7 +24,7 @@ import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; import org.apache.commons.collections.map.HashedMap; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.collect.Maps; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; diff --git a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java index ea51f46e4..7c02dba85 100644 --- a/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java +++ b/hbase/hbase-side/hbase-all-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAllSideInfo.java @@ -24,10 +24,11 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.util.ParseUtils; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; +import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; import java.util.List; @@ -52,7 +53,11 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { SqlNode conditionNode = joinInfo.getCondition(); List sqlNodeList = Lists.newArrayList(); - ParseUtils.parseAnd(conditionNode, sqlNodeList); + if(conditionNode.getKind() == SqlKind.AND){ + sqlNodeList.addAll(Lists.newArrayList(((SqlBasicCall)conditionNode).getOperands())); + }else{ + sqlNodeList.add(conditionNode); + } for(SqlNode sqlNode : sqlNodeList){ dealOneEqualCon(sqlNode, sideTableName); diff --git a/hbase/hbase-side/hbase-async-side/pom.xml b/hbase/hbase-side/hbase-async-side/pom.xml index d02498881..312d2c2ae 100644 --- a/hbase/hbase-side/hbase-async-side/pom.xml +++ b/hbase/hbase-side/hbase-async-side/pom.xml @@ -50,7 +50,6 @@ org.apache.hadoop:hadoop-common - org.apache.hadoop:hadoop-yarn-common org.apache.hadoop:hadoop-auth org.apache.hadoop:hadoop-mapreduce-client-core org.slf4j:* diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java index 05751d61e..6e82e4109 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncReqRow.java @@ -30,7 +30,7 @@ import com.dtstack.flink.sql.side.hbase.rowkeydealer.PreRowKeyModeDealerDealer; import com.dtstack.flink.sql.side.hbase.rowkeydealer.RowKeyEqualModeDealer; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; -import com.dtstack.flink.sql.factory.DTThreadFactory; +import com.dtstack.flink.sql.threadFactory.DTThreadFactory; import com.google.common.collect.Maps; import com.stumbleupon.async.Deferred; import org.apache.flink.api.java.typeutils.RowTypeInfo; @@ -129,7 +129,6 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except Object equalObj = input.getField(conValIndex); if(equalObj == null){ resultFuture.complete(null); - return; } refData.put(sideInfo.getEqualFieldList().get(i), equalObj); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncSideInfo.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncSideInfo.java index 2bfdd0d44..4ff25ad9c 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncSideInfo.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/HbaseAsyncSideInfo.java @@ -5,11 +5,12 @@ import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.hbase.table.HbaseSideTableInfo; -import com.dtstack.flink.sql.util.ParseUtils; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; import java.util.List; import java.util.Map; @@ -54,7 +55,11 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { SqlNode conditionNode = joinInfo.getCondition(); List sqlNodeList = Lists.newArrayList(); - ParseUtils.parseAnd(conditionNode, sqlNodeList); + if(conditionNode.getKind() == SqlKind.AND){ + sqlNodeList.addAll(Lists.newArrayList(((SqlBasicCall)conditionNode).getOperands())); + }else{ + sqlNodeList.add(conditionNode); + } for(SqlNode sqlNode : sqlNodeList){ dealOneEqualCon(sqlNode, sideTableName); diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java index d73b31e75..5fe61af4f 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/AbsRowKeyModeDealer.java @@ -23,7 +23,7 @@ import com.dtstack.flink.sql.side.FieldInfo; import com.dtstack.flink.sql.side.cache.AbsSideCache; import org.apache.calcite.sql.JoinType; -import com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.collect.Maps; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.types.Row; import org.hbase.async.HBaseClient; diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java index b75bca40c..d79f67ec5 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/PreRowKeyModeDealerDealer.java @@ -28,7 +28,7 @@ import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; -import com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.types.Row; import org.hbase.async.BinaryPrefixComparator; @@ -88,7 +88,6 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, } List cacheContent = Lists.newArrayList(); - List rowList = Lists.newArrayList(); for(List oneRow : args){ try { @@ -121,7 +120,8 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, if (openCache) { cacheContent.add(sideVal); } - rowList.add(row); + + resultFuture.complete(Collections.singleton(row)); } } catch (Exception e) { resultFuture.complete(null); @@ -130,10 +130,6 @@ private String dealOneRow(ArrayList> args, String rowKeyStr, } } - if (rowList.size() > 0){ - resultFuture.complete(rowList); - } - if(openCache){ sideCache.putCache(rowKeyStr, CacheObj.buildCacheObj(ECacheContentType.MultiLine, cacheContent)); } diff --git a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java index 8604db18e..4a4b60ff6 100644 --- a/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java +++ b/hbase/hbase-side/hbase-async-side/src/main/java/com/dtstack/flink/sql/side/hbase/rowkeydealer/RowKeyEqualModeDealer.java @@ -28,7 +28,7 @@ import com.dtstack.flink.sql.side.hbase.utils.HbaseUtils; import com.google.common.collect.Maps; import org.apache.calcite.sql.JoinType; -import com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.types.Row; import org.hbase.async.GetRequest; @@ -106,7 +106,7 @@ public void asyncGetData(String tableName, String rowKeyStr, Row input, ResultFu } } }catch (Exception e){ - resultFuture.completeExceptionally(e); + resultFuture.complete(null); LOG.error("record:" + input); LOG.error("get side record exception:", e); } diff --git a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java index 114b7fa6a..af2053fe6 100644 --- a/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java +++ b/hbase/hbase-side/hbase-side-core/src/main/java/com/dtstack/flink/sql/side/hbase/RowKeyBuilder.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.side.hbase; import com.dtstack.flink.sql.side.hbase.enums.EReplaceType; -import com.google.common.collect.Lists; +import org.apache.flink.shaded.curator.org.apache.curator.shaded.com.google.common.collect.Lists; import java.io.Serializable; import java.util.List; diff --git a/hbase/hbase-sink/pom.xml b/hbase/hbase-sink/pom.xml index 11ecf0a35..26ead3574 100644 --- a/hbase/hbase-sink/pom.xml +++ b/hbase/hbase-sink/pom.xml @@ -68,7 +68,7 @@ + tofile="${basedir}/../../plugins/hbasesink/${project.name}.jar" /> diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java index e88f34372..2ef2c6d52 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/HbaseOutputFormat.java @@ -64,8 +64,6 @@ public class HbaseOutputFormat extends MetricOutputFormat { public final SimpleDateFormat ROWKEY_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss"); public final SimpleDateFormat FIELD_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - private static int rowLenth = 1000; - @Override public void configure(Configuration parameters) { LOG.warn("---configure---"); @@ -135,10 +133,6 @@ public void writeRecord(Tuple2 tuple2) throws IOException { } table.put(put); - - if (outRecords.getCount()%rowLenth == 0){ - LOG.info(record.toString()); - } outRecords.inc(); } diff --git a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java index ed28f781d..4e286fd45 100644 --- a/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java +++ b/hbase/hbase-sink/src/main/java/com/dtstack/flink/sql/sink/hbase/table/HbaseTableInfo.java @@ -22,7 +22,7 @@ import com.dtstack.flink.sql.table.TargetTableInfo; -import com.google.common.base.Preconditions; +import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; import java.util.Map; /** diff --git a/kafka/kafka-sink/pom.xml b/kafka/kafka-sink/pom.xml deleted file mode 100644 index 41aa899c2..000000000 --- a/kafka/kafka-sink/pom.xml +++ /dev/null @@ -1,89 +0,0 @@ - - - - sql.kafka - com.dtstack.flink - 1.0-SNAPSHOT - ../pom.xml - - 4.0.0 - - sql.sink.kafka - 1.0-SNAPSHOT - kafka-sink - jar - - - - - org.apache.flink - flink-json - ${flink.version} - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 1.4 - - - package - - shade - - - - - org.slf4j - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - maven-antrun-plugin - 1.2 - - - copy-resources - - package - - run - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer.java deleted file mode 100644 index b7976a30e..000000000 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka; - -import com.dtstack.flink.sql.metric.MetricConstant; -import org.apache.flink.api.common.functions.RuntimeContext; -import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.metrics.Counter; -import org.apache.flink.metrics.MeterView; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; - -import java.util.Properties; - -/** - * @author: chuixue - * @create: 2019-11-05 11:54 - * @description: - **/ -public class CustomerFlinkKafkaProducer extends FlinkKafkaProducer { - - CustomerJsonRowSerializationSchema schema; - - public CustomerFlinkKafkaProducer(String topicId, SerializationSchema serializationSchema, Properties producerConfig) { - super(topicId, serializationSchema, producerConfig); - this.schema = (CustomerJsonRowSerializationSchema) serializationSchema; - } - - @Override - public void open(Configuration configuration) { - RuntimeContext ctx = getRuntimeContext(); - Counter counter = ctx.getMetricGroup().counter(MetricConstant.DT_NUM_RECORDS_OUT); - MeterView meter = ctx.getMetricGroup().meter(MetricConstant.DT_NUM_RECORDS_OUT_RATE, new MeterView(counter, 20)); - - schema.setCounter(counter); - - try { - super.open(configuration); - } catch (Exception e) { - throw new RuntimeException("",e); - } - } - -} diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java deleted file mode 100644 index d698d3ee2..000000000 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka; - -import org.apache.flink.annotation.PublicEvolving; -import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; -import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeinfo.Types; -import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.formats.json.JsonRowDeserializationSchema; -import org.apache.flink.formats.json.JsonRowSchemaConverter; -import org.apache.flink.metrics.Counter; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.sql.Time; -import java.sql.Timestamp; -import java.text.SimpleDateFormat; - -/** - * Serialization schema that serializes an object of Flink types into a JSON bytes. - * - *

Serializes the input Flink object into a JSON string and - * converts it into byte[]. - * - *

Result byte[] messages can be deserialized using {@link JsonRowDeserializationSchema}. - */ -@PublicEvolving -public class CustomerJsonRowSerializationSchema implements SerializationSchema { - - private static final long serialVersionUID = -2885556750743978636L; - - /** Type information describing the input type. */ - private final TypeInformation typeInfo; - - /** Object mapper that is used to create output JSON objects. */ - private final ObjectMapper mapper = new ObjectMapper(); - - /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone, without milliseconds). */ - private SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm:ss'Z'"); - - /** Formatter for RFC 3339-compliant string representation of a time value (with UTC timezone). */ - private SimpleDateFormat timeFormatWithMillis = new SimpleDateFormat("HH:mm:ss.SSS'Z'"); - - /** Formatter for RFC 3339-compliant string representation of a timestamp value (with UTC timezone). */ - private SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - - /** Reusable object node. */ - private transient ObjectNode node; - - private Counter counter; - - /** - * Creates a JSON serialization schema for the given type information. - * - * @param typeInfo The field names of {@link Row} are used to map to JSON properties. - */ - public CustomerJsonRowSerializationSchema(TypeInformation typeInfo) { - Preconditions.checkNotNull(typeInfo, "Type information"); - this.typeInfo = typeInfo; - } - - /** - * Creates a JSON serialization schema for the given JSON schema. - * - * @param jsonSchema JSON schema describing the result type - * - * @see http://json-schema.org/ - */ - public CustomerJsonRowSerializationSchema(String jsonSchema) { - this(JsonRowSchemaConverter.convert(jsonSchema)); - } - - @Override - public byte[] serialize(Row row) { - if (node == null) { - node = mapper.createObjectNode(); - } - - try { - convertRow(node, (RowTypeInfo) typeInfo, row); - counter.inc(); - return mapper.writeValueAsBytes(node); - } catch (Throwable t) { - throw new RuntimeException("Could not serialize row '" + row + "'. " + - "Make sure that the schema matches the input.", t); - } - } - - // -------------------------------------------------------------------------------------------- - - private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { - if (reuse == null) { - reuse = mapper.createObjectNode(); - } - final String[] fieldNames = info.getFieldNames(); - final TypeInformation[] fieldTypes = info.getFieldTypes(); - - // validate the row - if (row.getArity() != fieldNames.length) { - throw new IllegalStateException(String.format( - "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); - } - - for (int i = 0; i < fieldNames.length; i++) { - final String name = fieldNames[i]; - - final JsonNode fieldConverted = convert(reuse, reuse.get(name), fieldTypes[i], row.getField(i)); - reuse.set(name, fieldConverted); - } - - return reuse; - } - - private JsonNode convert(ContainerNode container, JsonNode reuse, TypeInformation info, Object object) { - if (info == Types.VOID || object == null) { - return container.nullNode(); - } else if (info == Types.BOOLEAN) { - return container.booleanNode((Boolean) object); - } else if (info == Types.STRING) { - return container.textNode((String) object); - } else if (info == Types.BIG_DEC) { - // convert decimal if necessary - if (object instanceof BigDecimal) { - return container.numberNode((BigDecimal) object); - } - return container.numberNode(BigDecimal.valueOf(((Number) object).doubleValue())); - } else if (info == Types.BIG_INT) { - // convert integer if necessary - if (object instanceof BigInteger) { - return container.numberNode((BigInteger) object); - } - return container.numberNode(BigInteger.valueOf(((Number) object).longValue())); - } else if (info == Types.SQL_DATE) { - return container.textNode(object.toString()); - } else if (info == Types.SQL_TIME) { - final Time time = (Time) object; - // strip milliseconds if possible - if (time.getTime() % 1000 > 0) { - return container.textNode(timeFormatWithMillis.format(time)); - } - return container.textNode(timeFormat.format(time)); - } else if (info == Types.SQL_TIMESTAMP) { - return container.textNode(timestampFormat.format((Timestamp) object)); - } else if (info instanceof RowTypeInfo) { - if (reuse != null && reuse instanceof ObjectNode) { - return convertRow((ObjectNode) reuse, (RowTypeInfo) info, (Row) object); - } else { - return convertRow(null, (RowTypeInfo) info, (Row) object); - } - } else if (info instanceof ObjectArrayTypeInfo) { - if (reuse != null && reuse instanceof ArrayNode) { - return convertObjectArray((ArrayNode) reuse, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); - } else { - return convertObjectArray(null, ((ObjectArrayTypeInfo) info).getComponentInfo(), (Object[]) object); - } - } else if (info instanceof BasicArrayTypeInfo) { - if (reuse != null && reuse instanceof ArrayNode) { - return convertObjectArray((ArrayNode) reuse, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); - } else { - return convertObjectArray(null, ((BasicArrayTypeInfo) info).getComponentInfo(), (Object[]) object); - } - } else if (info instanceof PrimitiveArrayTypeInfo && ((PrimitiveArrayTypeInfo) info).getComponentType() == Types.BYTE) { - return container.binaryNode((byte[]) object); - } else { - // for types that were specified without JSON schema - // e.g. POJOs - try { - return mapper.valueToTree(object); - } catch (IllegalArgumentException e) { - throw new IllegalStateException("Unsupported type information '" + info + "' for object: " + object, e); - } - } - } - - private ArrayNode convertObjectArray(ArrayNode reuse, TypeInformation info, Object[] array) { - if (reuse == null) { - reuse = mapper.createArrayNode(); - } else { - reuse.removeAll(); - } - - for (Object object : array) { - reuse.add(convert(reuse, null, info, object)); - } - return reuse; - } - - public Counter getCounter() { - return counter; - } - - public void setCounter(Counter counter) { - this.counter = counter; - } -} diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafkaJsonTableSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafkaJsonTableSink.java deleted file mode 100644 index af6e54854..000000000 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafkaJsonTableSink.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka; - -import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.functions.sink.SinkFunction; -import org.apache.flink.streaming.connectors.kafka.KafkaTableSink; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.utils.TableConnectorUtils; -import org.apache.flink.types.Row; - -import java.util.Optional; -import java.util.Properties; - -/** - * @author: chuixue - * @create: 2019-11-05 11:54 - * @description: - **/ -public class CustomerKafkaJsonTableSink extends KafkaTableSink { - - protected SerializationSchema schema; - - - public CustomerKafkaJsonTableSink(TableSchema schema, - String topic, - Properties properties, - Optional> partitioner, - SerializationSchema serializationSchema) { - - super(schema, topic, properties, partitioner, serializationSchema); - this.schema = serializationSchema; - } - - @Override - protected SinkFunction createKafkaProducer(String topic, Properties properties, SerializationSchema serializationSchema, Optional> optional) { - return new CustomerFlinkKafkaProducer(topic, serializationSchema, properties); - } - - @Override - public void emitDataStream(DataStream dataStream) { - SinkFunction kafkaProducer = createKafkaProducer(topic, properties, schema, partitioner); - // always enable flush on checkpoint to achieve at-least-once if query runs with checkpointing enabled. - //kafkaProducer.setFlushOnCheckpoint(true); - dataStream.addSink(kafkaProducer).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - } -} diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java deleted file mode 100644 index 44bf9f98b..000000000 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka; - -import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.KafkaTableSinkBase; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.sinks.RetractStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.types.Row; - -import java.util.Optional; -import java.util.Properties; - -/** - * @author: chuixue - * @create: 2019-11-05 11:45 - * @description: - **/ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - protected String topic; - - protected int parallelism; - - protected Properties properties; - - /** Serialization schema for encoding records to Kafka. */ - protected SerializationSchema serializationSchema; - - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - - @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { - KafkaSinkTableInfo kafkaSinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafkaSinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafkaSinkTableInfo.getBootstrapServers()); - - for (String key : kafkaSinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafkaSinkTableInfo.getKafkaParam(key)); - } - this.partitioner = Optional.of(new FlinkFixedPartitioner<>()); - this.fieldNames = kafkaSinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafkaSinkTableInfo.getFields().length]; - for (int i = 0; i < kafkaSinkTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - KafkaTableSinkBase kafkaTableSink = new CustomerKafkaJsonTableSink( - schema, - topic, - properties, - partitioner, - serializationSchema - ); - - DataStream ds = dataStream.map((Tuple2 record) -> { - return record.f1; - }).returns(getOutputType().getTypeAt(1)).setParallelism(parallelism); - - kafkaTableSink.emitDataStream(ds); - } - - @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } -} diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java deleted file mode 100644 index f633c8112..000000000 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka.table; - -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.util.MathUtil; - -import java.util.Map; - -/** - * @author: chuixue - * @create: 2019-11-05 11:46 - * @description: - **/ -public class KafkaSinkParser extends AbsTableParser { - @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { - KafkaSinkTableInfo kafkaSinkTableInfo = new KafkaSinkTableInfo(); - kafkaSinkTableInfo.setName(tableName); - parseFieldsInfo(fieldsInfo, kafkaSinkTableInfo); - kafkaSinkTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(KafkaSinkTableInfo.PARALLELISM_KEY.toLowerCase()))); - - if (props.get(KafkaSinkTableInfo.SINK_DATA_TYPE) != null) { - kafkaSinkTableInfo.setSinkDataType(props.get(KafkaSinkTableInfo.SINK_DATA_TYPE).toString()); - } - -// if (props.get(KafkaSinkTableInfo.FIELD_DELINITER) != null) { -// kafka11SinkTableInfo.setFieldDelimiter(props.get(KafkaSinkTableInfo.FIELD_DELINITER).toString()); -// } - - kafkaSinkTableInfo.setBootstrapServers(MathUtil.getString(props.get(KafkaSinkTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase()))); - kafkaSinkTableInfo.setTopic(MathUtil.getString(props.get(KafkaSinkTableInfo.TOPIC_KEY.toLowerCase()))); - - Integer parallelism = MathUtil.getIntegerVal(props.get(KafkaSinkTableInfo.PARALLELISM_KEY.toLowerCase())); - kafkaSinkTableInfo.setParallelism(parallelism); - - for (String key : props.keySet()) { - if (!key.isEmpty() && key.startsWith("kafka.")) { - kafkaSinkTableInfo.addKafkaParam(key.substring(6), props.get(key).toString()); - } - } - kafkaSinkTableInfo.check(); - - return kafkaSinkTableInfo; - } -} diff --git a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java b/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java deleted file mode 100644 index 1d23932c1..000000000 --- a/kafka/kafka-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka.table; - -import com.dtstack.flink.sql.table.TargetTableInfo; -import com.google.common.base.Preconditions; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * @author: chuixue - * @create: 2019-11-05 11:46 - * @description: - **/ -public class KafkaSinkTableInfo extends TargetTableInfo { - //version - private static final String CURR_TYPE = "kafka"; - - public KafkaSinkTableInfo(){ - super.setType(CURR_TYPE); - } - public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; - - public static final String TOPIC_KEY = "topic"; - - private String bootstrapServers; - - public Map kafkaParam = new HashMap(); - - private String topic; - - - public void addKafkaParam(String key,String value){ - kafkaParam.put(key,value); - } - - public String getKafkaParam(String key){ - return kafkaParam.get(key); - } - - public Set getKafkaParamKeys(){ - return kafkaParam.keySet(); - } - - - public String getBootstrapServers() { - return bootstrapServers; - } - - public void setBootstrapServers(String bootstrapServers) { - this.bootstrapServers = bootstrapServers; - } - - public String getTopic() { - return topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - - @Override - public boolean check() { - Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); - Preconditions.checkNotNull(topic, "kafka of topic is required"); - //Preconditions.checkNotNull(kafkaParam.get("groupId"), "kafka of groupId is required"); - return false; - } - - @Override - public String getType() { - return super.getType(); - } -} diff --git a/kafka/kafka-source/pom.xml b/kafka/kafka-source/pom.xml deleted file mode 100644 index 55ca950fb..000000000 --- a/kafka/kafka-source/pom.xml +++ /dev/null @@ -1,81 +0,0 @@ - - - - sql.kafka - com.dtstack.flink - 1.0-SNAPSHOT - ../pom.xml - - 4.0.0 - - sql.source.kafka - jar - - kafka-source - http://maven.apache.org - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 1.4 - - - package - - shade - - - - - org.slf4j - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - maven-antrun-plugin - 1.2 - - - copy-resources - - package - - run - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java deleted file mode 100644 index 6d3e57957..000000000 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.source.kafka; - -import com.dtstack.flink.sql.source.AbsDeserialization; -import com.dtstack.flink.sql.source.kafka.metric.KafkaTopicPartitionLagMetric; -import com.dtstack.flink.sql.table.TableInfo; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeinfo.Types; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.calcite.shaded.com.google.common.base.Strings; -import org.apache.flink.metrics.MetricGroup; -import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.JsonNodeType; -import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.TextNode; -import org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread; -import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; -import org.apache.flink.types.Row; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.consumer.internals.SubscriptionState; -import org.apache.kafka.common.TopicPartition; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static com.dtstack.flink.sql.metric.MetricConstant.*; - -/** - * @author: chuixue - * @create: 2019-11-05 10:57 - * @description: - **/ -public class CustomerJsonDeserialization extends AbsDeserialization { - - private static final Logger LOG = LoggerFactory.getLogger(CustomerJsonDeserialization.class); - - private static final long serialVersionUID = 2385115520960444192L; - - private static int dirtyDataFrequency = 1000; - - private final ObjectMapper objectMapper = new ObjectMapper(); - - /** Type information describing the result type. */ - private final TypeInformation typeInfo; - - /** Field names to parse. Indices match fieldTypes indices. */ - private final String[] fieldNames; - - /** Types to parse fields as. Indices match fieldNames indices. */ - private final TypeInformation[] fieldTypes; - - private AbstractFetcher fetcher; - - private boolean firstMsg = true; - - private Map nodeAndJsonNodeMapping = Maps.newHashMap(); - - private Map rowAndFieldMapping; - - private List fieldExtraInfos; - - public CustomerJsonDeserialization(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos){ - this.typeInfo = typeInfo; - this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); - this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); - this.rowAndFieldMapping= rowAndFieldMapping; - this.fieldExtraInfos = fieldExtraInfos; - } - - @Override - public Row deserialize(byte[] message) throws IOException { - - if(firstMsg){ - try { - registerPtMetric(fetcher); - } catch (Exception e) { - LOG.error("register topic partition metric error.", e); - } - - firstMsg = false; - } - - try { - JsonNode root = objectMapper.readTree(message); - - if (numInRecord.getCount() % dirtyDataFrequency == 0) { - LOG.info(root.toString()); - } - - numInRecord.inc(); - numInBytes.inc(message.length); - - parseTree(root, null); - Row row = new Row(fieldNames.length); - - for (int i = 0; i < fieldNames.length; i++) { - JsonNode node = getIgnoreCase(fieldNames[i]); - TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); - - if (node == null) { - if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { - throw new IllegalStateException("Failed to find field with name '" - + fieldNames[i] + "'."); - } else { - row.setField(i, null); - } - } else { - // Read the value as specified type - - Object value = convert(node, fieldTypes[i]); - row.setField(i, value); - } - } - - numInResolveRecord.inc(); - return row; - } catch (Exception e) { - //add metric of dirty data - if (dirtyDataCounter.getCount() % dirtyDataFrequency == 0) { - LOG.info("dirtyData: " + new String(message)); - LOG.error("" , e); - } - dirtyDataCounter.inc(); - return null; - }finally { - nodeAndJsonNodeMapping.clear(); - } - } - - public JsonNode getIgnoreCase(String key) { - String nodeMappingKey = rowAndFieldMapping.getOrDefault(key, key); - return nodeAndJsonNodeMapping.get(nodeMappingKey); - } - - private void parseTree(JsonNode jsonNode, String prefix){ - if (jsonNode.isArray()) { - ArrayNode array = (ArrayNode) jsonNode; - for (int i = 0; i < array.size(); i++) { - JsonNode child = array.get(i); - String nodeKey = getNodeKey(prefix, i); - - if (child.isValueNode()) { - nodeAndJsonNodeMapping.put(nodeKey, child); - } else { - if (rowAndFieldMapping.containsValue(nodeKey)) { - nodeAndJsonNodeMapping.put(nodeKey, child); - } - parseTree(child, nodeKey); - } - } - return; - } - - Iterator iterator = jsonNode.fieldNames(); - while (iterator.hasNext()){ - String next = iterator.next(); - JsonNode child = jsonNode.get(next); - String nodeKey = getNodeKey(prefix, next); - - if (child.isValueNode()){ - nodeAndJsonNodeMapping.put(nodeKey, child); - }else if(child.isArray()){ - parseTree(child, nodeKey); - }else { - parseTree(child, nodeKey); - } - } - } - - private String getNodeKey(String prefix, String nodeName){ - if(Strings.isNullOrEmpty(prefix)){ - return nodeName; - } - - return prefix + "." + nodeName; - } - - private String getNodeKey(String prefix, int i) { - if (Strings.isNullOrEmpty(prefix)) { - return "[" + i + "]"; - } - return prefix + "[" + i + "]"; - } - - public void setFetcher(AbstractFetcher fetcher) { - this.fetcher = fetcher; - } - - protected void registerPtMetric(AbstractFetcher fetcher) throws Exception { - - Field consumerThreadField = fetcher.getClass().getSuperclass().getDeclaredField("consumerThread"); - consumerThreadField.setAccessible(true); - KafkaConsumerThread consumerThread = (KafkaConsumerThread) consumerThreadField.get(fetcher); - - Field hasAssignedPartitionsField = consumerThread.getClass().getDeclaredField("hasAssignedPartitions"); - hasAssignedPartitionsField.setAccessible(true); - - //wait until assignedPartitions - - boolean hasAssignedPartitions = (boolean) hasAssignedPartitionsField.get(consumerThread); - - if(!hasAssignedPartitions){ - throw new RuntimeException("wait 50 secs, but not assignedPartitions"); - } - - Field consumerField = consumerThread.getClass().getDeclaredField("consumer"); - consumerField.setAccessible(true); - - KafkaConsumer kafkaConsumer = (KafkaConsumer) consumerField.get(consumerThread); - Field subscriptionStateField = kafkaConsumer.getClass().getDeclaredField("subscriptions"); - subscriptionStateField.setAccessible(true); - - //topic partitions lag - SubscriptionState subscriptionState = (SubscriptionState) subscriptionStateField.get(kafkaConsumer); - Set assignedPartitions = subscriptionState.assignedPartitions(); - for(TopicPartition topicPartition : assignedPartitions){ - MetricGroup metricGroup = getRuntimeContext().getMetricGroup().addGroup(DT_TOPIC_GROUP, topicPartition.topic()) - .addGroup(DT_PARTITION_GROUP, topicPartition.partition() + ""); - metricGroup.gauge(DT_TOPIC_PARTITION_LAG_GAUGE, new KafkaTopicPartitionLagMetric(subscriptionState, topicPartition)); - } - - } - - private static String partitionLagMetricName(TopicPartition tp) { - return tp + ".records-lag"; - } - - private Object convert(JsonNode node, TypeInformation info) { - if (info.getTypeClass().equals(Types.BOOLEAN.getTypeClass())) { - return node.asBoolean(); - } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { - return node.asText(); - } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { - return Date.valueOf(node.asText()); - } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { - // local zone - return Time.valueOf(node.asText()); - } else if (info.getTypeClass().equals(Types.SQL_TIMESTAMP.getTypeClass())) { - // local zone - return Timestamp.valueOf(node.asText()); - } else { - // for types that were specified without JSON schema - // e.g. POJOs - try { - return objectMapper.treeToValue(node, info.getTypeClass()); - } catch (JsonProcessingException e) { - throw new IllegalStateException("Unsupported type information '" + info + "' for node: " + node); - } - } - } -} diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafkaConsumer.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafkaConsumer.java deleted file mode 100644 index 7d35a35b3..000000000 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafkaConsumer.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.source.kafka; - -import com.dtstack.flink.sql.source.AbsDeserialization; -import org.apache.flink.metrics.MetricGroup; -import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks; -import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks; -import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; -import org.apache.flink.streaming.connectors.kafka.config.OffsetCommitMode; -import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; -import org.apache.flink.types.Row; -import org.apache.flink.util.SerializedValue; - -import java.util.Arrays; -import java.util.Map; -import java.util.Properties; -import java.util.regex.Pattern; - -/** - * @author: chuixue - * @create: 2019-11-05 10:58 - * @description: - **/ -public class CustomerKafkaConsumer extends FlinkKafkaConsumer { - - private static final long serialVersionUID = -2265366268827807739L; - - private CustomerJsonDeserialization customerJsonDeserialization; - - public CustomerKafkaConsumer(String topic, AbsDeserialization valueDeserializer, Properties props) { - super(Arrays.asList(topic.split(",")), valueDeserializer, props); - this.customerJsonDeserialization = (CustomerJsonDeserialization) valueDeserializer; - } - - public CustomerKafkaConsumer(Pattern subscriptionPattern, AbsDeserialization valueDeserializer, Properties props) { - super(subscriptionPattern, valueDeserializer, props); - this.customerJsonDeserialization = (CustomerJsonDeserialization) valueDeserializer; - } - - @Override - public void run(SourceContext sourceContext) throws Exception { - customerJsonDeserialization.setRuntimeContext(getRuntimeContext()); - customerJsonDeserialization.initMetric(); - super.run(sourceContext); - } - - @Override - protected AbstractFetcher createFetcher(SourceContext sourceContext, Map assignedPartitionsWithInitialOffsets, SerializedValue> watermarksPeriodic, SerializedValue> watermarksPunctuated, StreamingRuntimeContext runtimeContext, OffsetCommitMode offsetCommitMode, MetricGroup consumerMetricGroup, boolean useMetrics) throws Exception { - AbstractFetcher fetcher = super.createFetcher(sourceContext, assignedPartitionsWithInitialOffsets, watermarksPeriodic, watermarksPunctuated, runtimeContext, offsetCommitMode, consumerMetricGroup, useMetrics); - customerJsonDeserialization.setFetcher(fetcher); - return fetcher; - } -} diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java deleted file mode 100644 index c26d99b62..000000000 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.source.kafka; - -import com.dtstack.flink.sql.source.IStreamSourceGener; -import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStreamSource; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.types.Row; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.regex.Pattern; - -/** - * @author: chuixue - * @create: 2019-11-05 10:55 - * @description: - **/ -public class KafkaSource implements IStreamSourceGener { - - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") - @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - - KafkaSourceTableInfo kafkaSourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafkaSourceTableInfo.getTopic(); - - Properties props = new Properties(); - props.setProperty("bootstrap.servers", kafkaSourceTableInfo.getBootstrapServers()); - if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) { - props.setProperty("auto.offset.reset", "none"); - } else { - props.setProperty("auto.offset.reset", kafkaSourceTableInfo.getOffsetReset()); - } - if (StringUtils.isNotBlank(kafkaSourceTableInfo.getGroupId())) { - props.setProperty("group.id", kafkaSourceTableInfo.getGroupId()); - } - - TypeInformation[] types = new TypeInformation[kafkaSourceTableInfo.getFields().length]; - for (int i = 0; i < kafkaSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafkaSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafkaSourceTableInfo.getFields()); - - FlinkKafkaConsumer kafkaSrc; - if (BooleanUtils.isTrue(kafkaSourceTableInfo.getTopicIsPattern())) { - kafkaSrc = new CustomerKafkaConsumer(Pattern.compile(topicName), - new CustomerJsonDeserialization(typeInformation, kafkaSourceTableInfo.getPhysicalFields(), kafkaSourceTableInfo.getFieldExtraInfoList()), props); - } else { - kafkaSrc = new CustomerKafkaConsumer(topicName, - new CustomerJsonDeserialization(typeInformation, kafkaSourceTableInfo.getPhysicalFields(), kafkaSourceTableInfo.getFieldExtraInfoList()), props); - } - - //earliest,latest - if ("earliest".equalsIgnoreCase(kafkaSourceTableInfo.getOffsetReset())) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(kafkaSourceTableInfo.getOffsetReset())) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(kafkaSourceTableInfo.getOffsetReset(), Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + kafkaSourceTableInfo.getOffsetReset()); - } - } else { - kafkaSrc.setStartFromLatest(); - } - - String fields = StringUtils.join(kafkaSourceTableInfo.getFields(), ","); - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafkaSourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } - return tableEnv.fromDataStream(kafkaSource, fields); - } -} diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/metric/KafkaTopicPartitionLagMetric.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/metric/KafkaTopicPartitionLagMetric.java deleted file mode 100644 index 7810056d7..000000000 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/metric/KafkaTopicPartitionLagMetric.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.source.kafka.metric; - -import org.apache.flink.metrics.Gauge; -import org.apache.kafka.clients.consumer.internals.SubscriptionState; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.requests.IsolationLevel; - -/** - * @author: chuixue - * @create: 2019-11-05 11:09 - * @description: - **/ -public class KafkaTopicPartitionLagMetric implements Gauge { - - private SubscriptionState subscriptionState; - - private TopicPartition tp; - - public KafkaTopicPartitionLagMetric(SubscriptionState subscriptionState, TopicPartition tp){ - this.subscriptionState = subscriptionState; - this.tp = tp; - } - - @Override - public Long getValue() { - return subscriptionState.partitionLag(tp, IsolationLevel.READ_UNCOMMITTED); - } -} diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java deleted file mode 100644 index a99f49298..000000000 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.source.kafka.table; - -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.util.ClassUtil; -import com.dtstack.flink.sql.util.MathUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @author: chuixue - * @create: 2019-11-05 11:08 - * @description: - **/ -public class KafkaSourceParser extends AbsSourceParser { - - private static final Logger LOG = LoggerFactory.getLogger(KafkaSourceParser.class); - - private static final String KAFKA_NEST_FIELD_KEY = "nestFieldKey"; - - private static Pattern kafkaNestFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(\\w+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); - - static { - keyPatternMap.put(KAFKA_NEST_FIELD_KEY, kafkaNestFieldKeyPattern); - - keyHandlerMap.put(KAFKA_NEST_FIELD_KEY, KafkaSourceParser::dealNestField); - } - - /** - * add parser for alias field - * - * @param matcher - * @param tableInfo - */ - static void dealNestField(Matcher matcher, TableInfo tableInfo) { - String physicalField = matcher.group(1); - String fieldType = matcher.group(3); - String mappingField = matcher.group(4); - Class fieldClass = ClassUtil.stringConvertClass(fieldType); - boolean notNull = matcher.group(5) != null; - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); - fieldExtraInfo.setNotNull(notNull); - - tableInfo.addPhysicalMappings(mappingField, physicalField); - tableInfo.addField(mappingField); - tableInfo.addFieldClass(fieldClass); - tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(fieldExtraInfo); - if (LOG.isInfoEnabled()) { - LOG.info(physicalField + "--->" + mappingField + " Class: " + fieldClass.toString()); - } - } - - @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) throws Exception { - KafkaSourceTableInfo kafkaSourceTableInfo = new KafkaSourceTableInfo(); - kafkaSourceTableInfo.setName(tableName); - parseFieldsInfo(fieldsInfo, kafkaSourceTableInfo); - - kafkaSourceTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(KafkaSourceTableInfo.PARALLELISM_KEY.toLowerCase()))); - String bootstrapServer = MathUtil.getString(props.get(KafkaSourceTableInfo.BOOTSTRAPSERVERS_KEY.toLowerCase())); - if (bootstrapServer == null || bootstrapServer.trim().equals("")) { - throw new Exception("BootstrapServers can not be empty!"); - } else { - kafkaSourceTableInfo.setBootstrapServers(bootstrapServer); - } - kafkaSourceTableInfo.setGroupId(MathUtil.getString(props.get(KafkaSourceTableInfo.GROUPID_KEY.toLowerCase()))); - kafkaSourceTableInfo.setTopic(MathUtil.getString(props.get(KafkaSourceTableInfo.TOPIC_KEY.toLowerCase()))); - kafkaSourceTableInfo.setOffsetReset(MathUtil.getString(props.get(KafkaSourceTableInfo.OFFSETRESET_KEY.toLowerCase()))); - kafkaSourceTableInfo.setTopicIsPattern(MathUtil.getBoolean(props.get(KafkaSourceTableInfo.TOPICISPATTERN_KEY.toLowerCase()))); - kafkaSourceTableInfo.setTimeZone(MathUtil.getString(props.get(KafkaSourceTableInfo.TIME_ZONE_KEY.toLowerCase()))); - kafkaSourceTableInfo.check(); - return kafkaSourceTableInfo; - } -} diff --git a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java deleted file mode 100644 index 33b704ac0..000000000 --- a/kafka/kafka-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.source.kafka.table; - -import com.dtstack.flink.sql.table.SourceTableInfo; -import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; - -/** - * @author: chuixue - * @create: 2019-11-05 11:09 - * @description: - **/ -public class KafkaSourceTableInfo extends SourceTableInfo { - - //version - private static final String CURR_TYPE = "kafka"; - - public static final String BOOTSTRAPSERVERS_KEY = "bootstrapServers"; - - public static final String TOPIC_KEY = "topic"; - - public static final String GROUPID_KEY = "groupId"; - - public static final String OFFSETRESET_KEY = "offsetReset"; - - public static final String TOPICISPATTERN_KEY = "topicIsPattern"; - - private String bootstrapServers; - - private String topic; - - private String groupId; - - private Boolean topicIsPattern = false; - - public Boolean getTopicIsPattern() { - return topicIsPattern; - } - - public void setTopicIsPattern(Boolean topicIsPattern) { - if (topicIsPattern == null) return; - - this.topicIsPattern = topicIsPattern; - } - - //latest, earliest - private String offsetReset = "latest"; - - private String offset; - - public KafkaSourceTableInfo() { - super.setType(CURR_TYPE); - } - - - public String getBootstrapServers() { - return bootstrapServers; - } - - public void setBootstrapServers(String bootstrapServers) { - this.bootstrapServers = bootstrapServers; - } - - public String getTopic() { - return topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - public String getGroupId() { - return groupId; - } - - public void setGroupId(String groupId) { - this.groupId = groupId; - } - - public String getOffsetReset() { - return offsetReset; - } - - public void setOffsetReset(String offsetReset) { - if (offsetReset == null) { - return; - } - - this.offsetReset = offsetReset; - } - - public String getOffset() { - return offset; - } - - public void setOffset(String offset) { - this.offset = offset; - } - - @Override - public boolean check() { - Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); - Preconditions.checkNotNull(topic, "kafka of topic is required"); - return false; - } - - @Override - public String getType() { -// return super.getType() + SOURCE_SUFFIX; - return super.getType(); - } -} diff --git a/kafka/pom.xml b/kafka/pom.xml deleted file mode 100644 index 772671ff6..000000000 --- a/kafka/pom.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - flink.sql - com.dtstack.flink - 1.0-SNAPSHOT - - 4.0.0 - - sql.kafka - pom - - - kafka-source - kafka-sink - - - - - org.apache.flink - flink-connector-kafka_2.11 - ${flink.version} - - - - junit - junit - 3.8.1 - test - - - - com.dtstack.flink - sql.core - 1.0-SNAPSHOT - provided - - - - - \ No newline at end of file diff --git a/kafka08/kafka08-sink/pom.xml b/kafka08/kafka08-sink/pom.xml deleted file mode 100644 index 47391d182..000000000 --- a/kafka08/kafka08-sink/pom.xml +++ /dev/null @@ -1,94 +0,0 @@ - - - - sql.kafka08 - com.dtstack.flink - 1.0-SNAPSHOT - ../pom.xml - - 4.0.0 - - sql.sink.kafka08 - jar - - kafka08-sink - http://maven.apache.org - - - - org.apache.flink - flink-connector-kafka-0.8_2.11 - ${flink.version} - - - org.apache.flink - flink-json - ${flink.version} - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 1.4 - - - package - - shade - - - - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - maven-antrun-plugin - 1.2 - - - copy-resources - - package - - run - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/kafka08/kafka08-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerCsvSerialization.java b/kafka08/kafka08-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerCsvSerialization.java deleted file mode 100644 index 4168edbd1..000000000 --- a/kafka08/kafka08-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerCsvSerialization.java +++ /dev/null @@ -1,123 +0,0 @@ -package com.dtstack.flink.sql.sink.kafka; - -import org.apache.commons.lang3.StringEscapeUtils; -import org.apache.flink.annotation.Internal; -import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; -import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataOutputView; -import org.apache.flink.types.Row; -import org.apache.flink.types.StringValue; - -import java.io.IOException; - -import static org.apache.flink.api.java.typeutils.runtime.NullMaskUtils.writeNullMask; - -/** - * - * Date: 2018/12/18 - * Company: www.dtstack.com - * @author DocLi - * - * @modifyer maqi - */ -@Internal -public final class CustomerCsvSerialization extends TypeSerializerSingleton { - - private static final long serialVersionUID = 1L; - - private String fieldDelimiter = "\u0001"; - private TypeInformation[] fieldTypes; - private TypeSerializer[] fieldSerializers; - private static final Row EMPTY = null; - - public CustomerCsvSerialization(String fielddelimiter,TypeInformation[] fieldTypes) { - this.fieldDelimiter = fielddelimiter; - this.fieldTypes = fieldTypes; - this.fieldSerializers = (TypeSerializer[])createSerializer(new ExecutionConfig()); - } - - public TypeSerializer[] createSerializer(ExecutionConfig config) { - int len = fieldTypes.length; - TypeSerializer[] fieldSerializers = new TypeSerializer[len]; - for (int i = 0; i < len; i++) { - fieldSerializers[i] = fieldTypes[i].createSerializer(config); - } - return fieldSerializers; - } - - @Override - public boolean isImmutableType() { - return true; - } - - @Override - public Row createInstance() { - return EMPTY; - } - - @Override - public Row copy(Row from) { - return null; - } - - @Override - public Row copy(Row from, Row reuse) { - return null; - } - - @Override - public int getLength() { - return -1; - } - - @Override - public void serialize(Row record, DataOutputView target) throws IOException { - int len = fieldSerializers.length; - - if (record.getArity() != len) { - throw new RuntimeException("Row arity of from does not match serializers."); - } - - // write a null mask - writeNullMask(len, record, target); - - // serialize non-null fields - StringBuffer stringBuffer = new StringBuffer(); - for (int i = 0; i < len; i++) { - Object o = record.getField(i); - if (o != null) { - //fieldSerializers[i].serialize(o, target); - stringBuffer.append(o); - } - if(i != len-1){ - stringBuffer.append(StringEscapeUtils.unescapeJava(fieldDelimiter)); - //fieldSerializers[i].serialize(fieldDelimiter, target); - } - } - StringValue.writeString(stringBuffer.toString(), target); - } - - @Override - public Row deserialize(DataInputView source) throws IOException { - return null; - } - - @Override - public Row deserialize(Row reuse, DataInputView source) throws IOException { - return null; - } - - @Override - public void copy(DataInputView source, DataOutputView target) throws IOException { - StringValue.copyString(source, target); - } - - @Override - public TypeSerializerSnapshot snapshotConfiguration() { - return null; - } -} diff --git a/kafka08/kafka08-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka08/kafka08-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java deleted file mode 100644 index fe3c00f03..000000000 --- a/kafka08/kafka08-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka; - -import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.kafka.table.KafkaSinkTableInfo; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.api.common.serialization.TypeInformationSerializationSchema; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.formats.json.JsonRowSerializationSchema; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.Kafka08TableSink; -import org.apache.flink.streaming.connectors.kafka.KafkaTableSinkBase; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.sinks.AppendStreamTableSink; -import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.types.Row; - -import java.util.Optional; -import java.util.Properties; -/** - * Date: 2018/12/18 - * Company: www.dtstack.com - * @author DocLi - * - * @modifyer maqi - */ -public class KafkaSink implements AppendStreamTableSink, IStreamSinkGener { - - protected String[] fieldNames; - - protected TypeInformation[] fieldTypes; - - /** The schema of the table. */ - private TableSchema schema; - - /** The Kafka topic to write to. */ - protected String topic; - - /** Properties for the Kafka producer. */ - protected Properties properties; - - /** Serialization schema for encoding records to Kafka. */ - protected SerializationSchema serializationSchema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - - @Override - public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { - KafkaSinkTableInfo kafka08SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; - this.topic = kafka08SinkTableInfo.getKafkaParam("topic"); - - Properties props = new Properties(); - for (String key:kafka08SinkTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafka08SinkTableInfo.getKafkaParam(key)); - } - this.properties = props; - this.partitioner = Optional.of(new FlinkFixedPartitioner<>()); - this.fieldNames = kafka08SinkTableInfo.getFields(); - TypeInformation[] types = new TypeInformation[kafka08SinkTableInfo.getFields().length]; - for(int i = 0; i< kafka08SinkTableInfo.getFieldClasses().length; i++){ - types[i] = TypeInformation.of(kafka08SinkTableInfo.getFieldClasses()[i]); - } - this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i dataStream) { - KafkaTableSinkBase kafkaTableSink = new Kafka08TableSink( - schema, - topic, - properties, - partitioner, - serializationSchema - ); - - kafkaTableSink.emitDataStream(dataStream); - } - - @Override - public TypeInformation getOutputType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public TypeInformation[] getFieldTypes() { - return fieldTypes; - } - - @Override - public TableSink configure(String[] fieldNames, TypeInformation[] fieldTypes) { - this.fieldNames = fieldNames; - this.fieldTypes = fieldTypes; - return this; - } - -} diff --git a/kafka08/kafka08-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka08/kafka08-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java deleted file mode 100644 index 2b6c50512..000000000 --- a/kafka08/kafka08-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka.table; - -import com.dtstack.flink.sql.table.AbsTableParser; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.util.MathUtil; - -import java.util.Map; - -/** - * - * Date: 2018/12/18 - * Company: www.dtstack.com - * - * @author DocLi - * - * @modifyer maqi - * - */ -public class KafkaSinkParser extends AbsTableParser { - @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - KafkaSinkTableInfo kafka08SinkTableInfo = new KafkaSinkTableInfo(); - kafka08SinkTableInfo.setName(tableName); - parseFieldsInfo(fieldsInfo, kafka08SinkTableInfo); - kafka08SinkTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(KafkaSinkTableInfo.PARALLELISM_KEY.toLowerCase()))); - if (props.get(KafkaSinkTableInfo.SINK_DATA_TYPE) != null) { - kafka08SinkTableInfo.setSinkDataType(props.get(KafkaSinkTableInfo.SINK_DATA_TYPE).toString()); - } - if (props.get(KafkaSinkTableInfo.FIELD_DELINITER) != null) { - kafka08SinkTableInfo.setFieldDelimiter(props.get(KafkaSinkTableInfo.FIELD_DELINITER).toString()); - } - - for (String key:props.keySet()) { - if (!key.isEmpty() && key.startsWith("kafka.")) { - kafka08SinkTableInfo.addKafkaParam(key.substring(6), props.get(key).toString()); - } - } - return kafka08SinkTableInfo; - } -} diff --git a/kafka08/kafka08-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java b/kafka08/kafka08-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java deleted file mode 100644 index 61acfa2d6..000000000 --- a/kafka08/kafka08-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkTableInfo.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.sink.kafka.table; - -import com.dtstack.flink.sql.table.TargetTableInfo; -import com.google.common.base.Preconditions; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * - * Date: 2018/12/18 - * Company: www.dtstack.com - * @author DocLi - * - * @modifyer maqi - * - */ -public class KafkaSinkTableInfo extends TargetTableInfo { - //version - private static final String CURR_TYPE = "kafka08"; - - public KafkaSinkTableInfo(){ - super.setType(CURR_TYPE); - } - - public Map kafkaParam = new HashMap(); - - public void addKafkaParam(String key,String value){ - kafkaParam.put(key,value); - } - - public String getKafkaParam(String key){ - return kafkaParam.get(key); - } - - public Set getKafkaParamKeys(){ - return kafkaParam.keySet(); - } - - @Override - public boolean check() { - Preconditions.checkNotNull(kafkaParam.get("bootstrap.servers"), "kafka of bootstrapServers is required"); - Preconditions.checkNotNull(kafkaParam.get("topic"), "kafka of topic is required"); - // Preconditions.checkNotNull(kafkaParam.get("groupId"), "kafka of groupId is required"); - return false; - } - - @Override - public String getType() { - return super.getType(); - } -} diff --git a/kafka08/kafka08-source/pom.xml b/kafka08/kafka08-source/pom.xml deleted file mode 100644 index 64264fd46..000000000 --- a/kafka08/kafka08-source/pom.xml +++ /dev/null @@ -1,91 +0,0 @@ - - - - sql.kafka08 - com.dtstack.flink - 1.0-SNAPSHOT - ../pom.xml - - 4.0.0 - - sql.source.kafka08 - jar - - kafka08-source - http://maven.apache.org - - - - org.apache.flink - flink-connector-kafka-0.8_2.11 - ${flink.version} - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 1.4 - - - package - - shade - - - - - org.slf4j - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - maven-antrun-plugin - 1.2 - - - copy-resources - - package - - run - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java deleted file mode 100644 index bfbffdf14..000000000 --- a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.source.kafka; - -import com.dtstack.flink.sql.source.IStreamSourceGener; -import com.dtstack.flink.sql.source.kafka.consumer.CustomerCommonConsumer; -import com.dtstack.flink.sql.source.kafka.consumer.CustomerCsvConsumer; -import com.dtstack.flink.sql.source.kafka.consumer.CustomerJsonConsumer; -import com.dtstack.flink.sql.source.kafka.deserialization.CustomerCommonDeserialization; -import com.dtstack.flink.sql.source.kafka.deserialization.CustomerCsvDeserialization; -import com.dtstack.flink.sql.source.kafka.deserialization.CustomerJsonDeserialization; -import com.dtstack.flink.sql.source.kafka.table.KafkaSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.PluginUtil; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStreamSource; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer08; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.types.Row; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.regex.Pattern; - -public class KafkaSource implements IStreamSourceGener
{ - - private static final String SOURCE_OPERATOR_NAME_TPL = "${topic}_${table}"; - - /** - * Get kafka data source, you need to provide the data field names, data types - * If you do not specify auto.offset.reset, the default use groupoffset - * - * @param sourceTableInfo - * @return - */ - @SuppressWarnings("rawtypes") - @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - KafkaSourceTableInfo kafka08SourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; - String topicName = kafka08SourceTableInfo.getKafkaParam("topic"); - String offsetReset = kafka08SourceTableInfo.getKafkaParam("auto.offset.reset"); - Boolean topicIsPattern = kafka08SourceTableInfo.getPatternTopic(); - - Properties props = new Properties(); - for (String key : kafka08SourceTableInfo.getKafkaParamKeys()) { - props.setProperty(key, kafka08SourceTableInfo.getKafkaParam(key)); - } - - TypeInformation[] types = new TypeInformation[kafka08SourceTableInfo.getFields().length]; - for (int i = 0; i < kafka08SourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(kafka08SourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, kafka08SourceTableInfo.getFields()); - - FlinkKafkaConsumer08 kafkaSrc; - String fields = StringUtils.join(kafka08SourceTableInfo.getFields(), ","); - - if ("json".equalsIgnoreCase(kafka08SourceTableInfo.getSourceDataType())) { - if (topicIsPattern) { - kafkaSrc = new CustomerJsonConsumer(Pattern.compile(topicName), - new com.dtstack.flink.sql.source.kafka.deserialization.CustomerJsonDeserialization(typeInformation), props); - } else { - kafkaSrc = new CustomerJsonConsumer(topicName, - new CustomerJsonDeserialization(typeInformation), props); - } - } else if ("csv".equalsIgnoreCase(kafka08SourceTableInfo.getSourceDataType())) { - if (topicIsPattern) { - kafkaSrc = new CustomerCsvConsumer(Pattern.compile(topicName), - new com.dtstack.flink.sql.source.kafka.deserialization.CustomerCsvDeserialization(typeInformation, - kafka08SourceTableInfo.getFieldDelimiter(), kafka08SourceTableInfo.getLengthCheckPolicy()), props); - } else { - kafkaSrc = new CustomerCsvConsumer(topicName, - new CustomerCsvDeserialization(typeInformation, - kafka08SourceTableInfo.getFieldDelimiter(), kafka08SourceTableInfo.getLengthCheckPolicy()), props); - } - } else { - if (topicIsPattern) { - kafkaSrc = new CustomerCommonConsumer(Pattern.compile(topicName), new com.dtstack.flink.sql.source.kafka.deserialization.CustomerCommonDeserialization(), props); - } else { - kafkaSrc = new CustomerCommonConsumer(topicName, new CustomerCommonDeserialization(), props); - } - } - - //earliest,latest - if ("earliest".equalsIgnoreCase(offsetReset)) { - kafkaSrc.setStartFromEarliest(); - } else if (DtStringUtil.isJosn(offsetReset)) {// {"0":12312,"1":12321,"2":12312} - try { - Properties properties = PluginUtil.jsonStrToObject(offsetReset, Properties.class); - Map offsetMap = PluginUtil.ObjectToMap(properties); - Map specificStartupOffsets = new HashMap<>(); - for (Map.Entry entry : offsetMap.entrySet()) { - specificStartupOffsets.put(new KafkaTopicPartition(topicName, Integer.valueOf(entry.getKey())), Long.valueOf(entry.getValue().toString())); - } - kafkaSrc.setStartFromSpecificOffsets(specificStartupOffsets); - } catch (Exception e) { - throw new RuntimeException("not support offsetReset type:" + offsetReset); - } - } else { - kafkaSrc.setStartFromLatest(); - } - String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafka08SourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } - return tableEnv.fromDataStream(kafkaSource, fields); - } -} diff --git a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/consumer/CustomerCommonConsumer.java b/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/consumer/CustomerCommonConsumer.java deleted file mode 100644 index 34b349e2c..000000000 --- a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/consumer/CustomerCommonConsumer.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.flink.sql.source.kafka.consumer; - -import com.dtstack.flink.sql.source.kafka.deserialization.CustomerCommonDeserialization; -import org.apache.flink.streaming.api.functions.source.SourceFunction; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer08; -import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema; -import org.apache.flink.types.Row; - -import java.util.Properties; -import java.util.regex.Pattern; - -/** - * - * Date: 2018/12/18 - * Company: www.dtstack.com - * @author DocLi - * - * @modifyer maqi - */ -public class CustomerCommonConsumer extends FlinkKafkaConsumer08 { - - private CustomerCommonDeserialization customerCommonDeserialization; - - - public CustomerCommonConsumer(String topic, KeyedDeserializationSchema deserializer, Properties props) { - super(topic, deserializer, props); - this.customerCommonDeserialization= (CustomerCommonDeserialization) deserializer; - } - - public CustomerCommonConsumer(Pattern subscriptionPattern, KeyedDeserializationSchema deserializer, Properties props) { - super(subscriptionPattern, deserializer, props); - this.customerCommonDeserialization= (CustomerCommonDeserialization) deserializer; - } - - - @Override - public void run(SourceFunction.SourceContext sourceContext) throws Exception { - customerCommonDeserialization.setRuntimeContext(getRuntimeContext()); - customerCommonDeserialization.initMetric(); - super.run(sourceContext); - } - -} diff --git a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/consumer/CustomerCsvConsumer.java b/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/consumer/CustomerCsvConsumer.java deleted file mode 100644 index 7dc95450e..000000000 --- a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/consumer/CustomerCsvConsumer.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.source.kafka.consumer; - -import com.dtstack.flink.sql.source.AbsDeserialization; -import com.dtstack.flink.sql.source.kafka.deserialization.CustomerCsvDeserialization; -import org.apache.flink.streaming.api.functions.source.SourceFunction; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer08; -import org.apache.flink.types.Row; - -import java.util.Arrays; -import java.util.Properties; -import java.util.regex.Pattern; - -/** - * Date: 2018/12/18 - * Company: www.dtstack.com - * @author DocLi - * - * @modifyer maqi - */ - -public class CustomerCsvConsumer extends FlinkKafkaConsumer08 { - - private static final long serialVersionUID = -2265366268827807739L; - - private CustomerCsvDeserialization customerCsvDeserialization; - - public CustomerCsvConsumer(String topic, AbsDeserialization valueDeserializer, Properties props) { - super(Arrays.asList(topic.split(",")), valueDeserializer, props); - this.customerCsvDeserialization = (CustomerCsvDeserialization) valueDeserializer; - } - - public CustomerCsvConsumer(Pattern subscriptionPattern, AbsDeserialization valueDeserializer, Properties props) { - super(subscriptionPattern, valueDeserializer, props); - this.customerCsvDeserialization = (CustomerCsvDeserialization) valueDeserializer; - } - - - - - @Override - public void run(SourceFunction.SourceContext sourceContext) throws Exception { - customerCsvDeserialization.setRuntimeContext(getRuntimeContext()); - customerCsvDeserialization.initMetric(); - super.run(sourceContext); - } - -} diff --git a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/consumer/CustomerJsonConsumer.java b/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/consumer/CustomerJsonConsumer.java deleted file mode 100644 index b627d81c0..000000000 --- a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/consumer/CustomerJsonConsumer.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.source.kafka.consumer; - -import com.dtstack.flink.sql.source.AbsDeserialization; -import com.dtstack.flink.sql.source.kafka.deserialization.CustomerJsonDeserialization; -import org.apache.flink.streaming.api.functions.source.SourceFunction; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer08; -import org.apache.flink.types.Row; - -import java.util.Arrays; -import java.util.Properties; -import java.util.regex.Pattern; - -/** - * Reason: - * Date: 2018/10/19 - * Company: www.dtstack.com - * - * @author xuchao - */ - -public class CustomerJsonConsumer extends FlinkKafkaConsumer08 { - - private static final long serialVersionUID = -2265366268827807739L; - - private CustomerJsonDeserialization customerJsonDeserialization; - - public CustomerJsonConsumer(String topic, AbsDeserialization valueDeserializer, Properties props) { - super(Arrays.asList(topic.split(",")), valueDeserializer, props); - this.customerJsonDeserialization = (CustomerJsonDeserialization) valueDeserializer; - } - - public CustomerJsonConsumer(Pattern subscriptionPattern, AbsDeserialization valueDeserializer, Properties props) { - super(subscriptionPattern, valueDeserializer, props); - this.customerJsonDeserialization = (CustomerJsonDeserialization) valueDeserializer; - } - - - @Override - public void run(SourceFunction.SourceContext sourceContext) throws Exception { - customerJsonDeserialization.setRuntimeContext(getRuntimeContext()); - customerJsonDeserialization.initMetric(); - super.run(sourceContext); - } - - -} diff --git a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/deserialization/CustomerCommonDeserialization.java b/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/deserialization/CustomerCommonDeserialization.java deleted file mode 100644 index c92ce4aa4..000000000 --- a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/deserialization/CustomerCommonDeserialization.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.source.kafka.deserialization; - -import com.dtstack.flink.sql.source.AbsDeserialization; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeinfo.Types; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TypeExtractor; -import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema; -import org.apache.flink.types.Row; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * Date: 2018/12/18 - * Company: www.dtstack.com - * @author DocLi - * - * @modifyer maqi - */ -public class CustomerCommonDeserialization extends AbsDeserialization implements KeyedDeserializationSchema { - private static final Logger LOG = LoggerFactory.getLogger(CustomerCommonDeserialization.class); - - public static final String[] KAFKA_COLUMNS = new String[]{"_TOPIC", "_MESSAGEKEY", "_MESSAGE", "_PARTITION", "_OFFSET"}; - - private boolean firstMsg = true; - - @Override - public Row deserialize(byte[] messageKey, byte[] message, String topic, int partition, long offset) { - - //numInRecord.inc(); - //numInBytes.inc(message.length); - //numInBytes.inc(messageKey.length); - - try { - Row row = Row.of( - topic, //topic - messageKey == null ? null : new String(messageKey, UTF_8), //key - new String(message, UTF_8), //message - partition, - offset - ); - return row; - } catch (Throwable t) { - LOG.error(t.getMessage()); - // dirtyDataCounter.inc(); - return null; - } - } - - @Override - public Row deserialize(byte[] message) throws IOException { - return null; - } - - - @Override - public boolean isEndOfStream(Row nextElement) { - return false; - } - - public TypeInformation getProducedType() { - TypeInformation[] types = new TypeInformation[]{ - TypeExtractor.createTypeInfo(String.class), - TypeExtractor.createTypeInfo(String.class), //createTypeInformation[String] - TypeExtractor.createTypeInfo(String.class), - Types.INT, - Types.LONG - }; - return new RowTypeInfo(types, KAFKA_COLUMNS); - } - -} diff --git a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/deserialization/CustomerCsvDeserialization.java b/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/deserialization/CustomerCsvDeserialization.java deleted file mode 100644 index f9f4c897c..000000000 --- a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/deserialization/CustomerCsvDeserialization.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flink.sql.source.kafka.deserialization; - - -import com.dtstack.flink.sql.source.AbsDeserialization; -import com.dtstack.flink.sql.util.DtStringUtil; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.types.Row; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; - -/** - * Date: 2018/12/18 - * Company: www.dtstack.com - * @author DocLi - * - * @modifyer maqi - */ - -public class CustomerCsvDeserialization extends AbsDeserialization { - - private static final Logger LOG = LoggerFactory.getLogger(CustomerCsvDeserialization.class); - - private static final long serialVersionUID = -2706012724306826506L; - - private final ObjectMapper objectMapper = new ObjectMapper(); - - /** Type information describing the result type. */ - private final TypeInformation typeInfo; - - /** Field names to parse. Indices match fieldTypes indices. */ - private final String[] fieldNames; - - /** Types to parse fields as. Indices match fieldNames indices. */ - private final TypeInformation[] fieldTypes; - - /** Flag indicating whether to fail on a missing field. */ - private boolean failOnMissingField; - - private String fieldDelimiter; - - private String lengthCheckPolicy; - - public CustomerCsvDeserialization(TypeInformation typeInfo, String fieldDelimiter, String lengthCheckPolicy){ - this.typeInfo = typeInfo; - - this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); - - this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); - - this.fieldDelimiter = fieldDelimiter; - - this.lengthCheckPolicy = lengthCheckPolicy; - } - - @Override - public Row deserialize(byte[] message) throws IOException { - - try { - //numInRecord.inc(); - //numInBytes.inc(message.length); - String[] fieldsList = null; - if (message != null && message.length > 0){ - fieldsList = new String(message).split(fieldDelimiter); - } - if (fieldsList == null || fieldsList.length != fieldNames.length){//exception condition - if (lengthCheckPolicy.equalsIgnoreCase("SKIP")) { - return null; - }else if (lengthCheckPolicy.equalsIgnoreCase("EXCEPTION")) { - throw new RuntimeException("lengthCheckPolicy Error,message have "+fieldsList.length+" fields,sql have "+fieldNames.length); - } - } - - Row row = new Row(fieldNames.length); - for (int i = 0; i < fieldNames.length; i++) { - if (i { - - private static final Logger LOG = LoggerFactory.getLogger(CustomerJsonDeserialization.class); - - private static final long serialVersionUID = 2385115520960444192L; - - private final ObjectMapper objectMapper = new ObjectMapper(); - - /** Type information describing the result type. */ - private final TypeInformation typeInfo; - - /** Field names to parse. Indices match fieldTypes indices. */ - private final String[] fieldNames; - - /** Types to parse fields as. Indices match fieldNames indices. */ - private final TypeInformation[] fieldTypes; - - /** Flag indicating whether to fail on a missing field. */ - private boolean failOnMissingField; - - private AbstractFetcher fetcher; - - public CustomerJsonDeserialization(TypeInformation typeInfo){ - this.typeInfo = typeInfo; - - this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); - - this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); - } - - @Override - public Row deserialize(byte[] message) throws IOException { - - try { - // numInRecord.inc(); - // numInBytes.inc(message.length); - - JsonNode root = objectMapper.readTree(message); - Row row = new Row(fieldNames.length); - for (int i = 0; i < fieldNames.length; i++) { - JsonNode node = getIgnoreCase(root, fieldNames[i]); - - if (node == null) { - if (failOnMissingField) { - throw new IllegalStateException("Failed to find field with name '" - + fieldNames[i] + "'."); - } else { - row.setField(i, null); - } - } else { - // Read the value as specified type - Object value = objectMapper.treeToValue(node, fieldTypes[i].getTypeClass()); - row.setField(i, value); - } - } - - // numInResolveRecord.inc(); - return row; - } catch (Throwable t) { - //add metric of dirty data - LOG.error(t.getMessage()); - // dirtyDataCounter.inc(); - return null; - } - } - - public void setFailOnMissingField(boolean failOnMissingField) { - this.failOnMissingField = failOnMissingField; - } - - public JsonNode getIgnoreCase(JsonNode jsonNode, String key) { - - Iterator iter = jsonNode.fieldNames(); - while (iter.hasNext()) { - String key1 = iter.next(); - if (key1.equalsIgnoreCase(key)) { - return jsonNode.get(key1); - } - } - - return null; - - } - - public void setFetcher(AbstractFetcher fetcher) { - this.fetcher = fetcher; - } - - - private static String partitionLagMetricName(TopicPartition tp) { - return tp + ".records-lag"; - } -} diff --git a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java deleted file mode 100644 index eb085ac78..000000000 --- a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flink.sql.source.kafka.table; - -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.util.MathUtil; - -import java.util.Map; -/** - * - * Date: 2018/12/18 - * Company: www.dtstack.com - * @author xuchao - * - * @modifyer DocLi - */ - -public class KafkaSourceParser extends AbsSourceParser { - - @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - - KafkaSourceTableInfo kafka08SourceTableInfo = new KafkaSourceTableInfo(); - kafka08SourceTableInfo.setName(tableName); - parseFieldsInfo(fieldsInfo, kafka08SourceTableInfo); - kafka08SourceTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(KafkaSourceTableInfo.PARALLELISM_KEY.toLowerCase()))); - - kafka08SourceTableInfo.setPatternTopic(MathUtil.getBoolean(props.get(KafkaSourceTableInfo.PATTERNTOPIC_KEY.toLowerCase()))); - - kafka08SourceTableInfo.setTimeZone(MathUtil.getString(props.get(KafkaSourceTableInfo.TIME_ZONE_KEY.toLowerCase()))); - - if (props.get(KafkaSourceTableInfo.SOURCE_DATA_TYPE) != null) { - kafka08SourceTableInfo.setSourceDataType(props.get(KafkaSourceTableInfo.SOURCE_DATA_TYPE).toString()); - } - if (props.get(KafkaSourceTableInfo.FIELD_DELINITER) != null) { - kafka08SourceTableInfo.setFieldDelimiter(props.get(KafkaSourceTableInfo.FIELD_DELINITER).toString()); - } - if (props.get(KafkaSourceTableInfo.LENGTH_CHECK_POLICY) != null) { - kafka08SourceTableInfo.setLengthCheckPolicy(props.get(KafkaSourceTableInfo.LENGTH_CHECK_POLICY).toString()); - } - for (String key:props.keySet()) { - if (!key.isEmpty() && key.startsWith("kafka.")) { - kafka08SourceTableInfo.addKafkaParam(key.substring(6), props.get(key).toString()); - } - } - return kafka08SourceTableInfo; - } -} diff --git a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java deleted file mode 100644 index 93e9d52f7..000000000 --- a/kafka08/kafka08-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -package com.dtstack.flink.sql.source.kafka.table; - -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.google.common.base.Preconditions; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - - -public class KafkaSourceTableInfo extends SourceTableInfo { - - //version - private static final String CURR_TYPE = "kafka08"; - - public static final String PATTERNTOPIC_KEY = "patterntopic"; - - private Boolean patternTopic=false; - - public Boolean getPatternTopic() { - return patternTopic; - } - - public void setPatternTopic(Boolean patternTopic) { - if (patternTopic==null){ - return; - } - this.patternTopic = patternTopic; - } - - public KafkaSourceTableInfo(){ - super.setType(CURR_TYPE); - } - - public Map kafkaParam = new HashMap<>(); - - public void addKafkaParam(String key,String value){ - kafkaParam.put(key,value); - } - - public String getKafkaParam(String key){ - return kafkaParam.get(key); - } - - public Set getKafkaParamKeys(){ - return kafkaParam.keySet(); - } - - @Override - public boolean check() { - Preconditions.checkNotNull(kafkaParam.get("bootstrap.servers"), "kafka of bootstrapServers is required"); - Preconditions.checkNotNull(kafkaParam.get("topic"), "kafka of topic is required"); - String offset = kafkaParam.get("auto.offset.reset"); - Preconditions.checkState(offset.equalsIgnoreCase("latest") - || offset.equalsIgnoreCase("earliest"), "kafka of offsetReset set fail"); - return false; - } - - @Override - public String getType() { - return super.getType(); - } -} diff --git a/kafka08/pom.xml b/kafka08/pom.xml deleted file mode 100644 index ced039c1c..000000000 --- a/kafka08/pom.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - flink.sql - com.dtstack.flink - 1.0-SNAPSHOT - - 4.0.0 - - sql.kafka08 - pom - - - kafka08-source - kafka08-sink - - - - - junit - junit - 3.8.1 - test - - - - com.dtstack.flink - sql.core - 1.0-SNAPSHOT - provided - - - - - \ No newline at end of file diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerCsvSerialization.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerCsvSerialization.java deleted file mode 100644 index bccf7ac9e..000000000 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerCsvSerialization.java +++ /dev/null @@ -1,120 +0,0 @@ -package com.dtstack.flink.sql.sink.kafka; - -import org.apache.commons.lang3.StringEscapeUtils; -import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; -import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataOutputView; -import org.apache.flink.types.Row; -import org.apache.flink.types.StringValue; - -import java.io.IOException; - -import static org.apache.flink.api.java.typeutils.runtime.NullMaskUtils.writeNullMask; -/** - * Date: 2018/12/18 - * Company: www.dtstack.com - * @author DocLi - * - * @modifyer maqi - * - */ -public final class CustomerCsvSerialization extends TypeSerializerSingleton { - - private static final long serialVersionUID = 1L; - - private String fieldDelimiter = "\u0001"; - private TypeInformation[] fieldTypes; - private TypeSerializer[] fieldSerializers; - private static final Row EMPTY = null; - - public CustomerCsvSerialization(String fielddelimiter,TypeInformation[] fieldTypes) { - this.fieldDelimiter = fielddelimiter; - this.fieldTypes = fieldTypes; - this.fieldSerializers = (TypeSerializer[])createSerializer(new ExecutionConfig()); - } - - public TypeSerializer[] createSerializer(ExecutionConfig config) { - int len = fieldTypes.length; - TypeSerializer[] fieldSerializers = new TypeSerializer[len]; - for (int i = 0; i < len; i++) { - fieldSerializers[i] = fieldTypes[i].createSerializer(config); - } - return fieldSerializers; - } - - @Override - public boolean isImmutableType() { - return true; - } - - @Override - public Row createInstance() { - return EMPTY; - } - - @Override - public Row copy(Row from) { - return null; - } - - @Override - public Row copy(Row from, Row reuse) { - return null; - } - - @Override - public int getLength() { - return -1; - } - - @Override - public void serialize(Row record, DataOutputView target) throws IOException { - int len = fieldSerializers.length; - - if (record.getArity() != len) { - throw new RuntimeException("Row arity of from does not match serializers."); - } - - // write a null mask - writeNullMask(len, record, target); - - // serialize non-null fields - StringBuffer stringBuffer = new StringBuffer(); - for (int i = 0; i < len; i++) { - Object o = record.getField(i); - if (o != null) { - //fieldSerializers[i].serialize(o, target); - stringBuffer.append(o); - } - if(i != len-1){ - stringBuffer.append(StringEscapeUtils.unescapeJava(fieldDelimiter)); - //fieldSerializers[i].serialize(fieldDelimiter, target); - } - } - StringValue.writeString(stringBuffer.toString(), target); - } - - @Override - public Row deserialize(DataInputView source) throws IOException { - return null; - } - - @Override - public Row deserialize(Row reuse, DataInputView source) throws IOException { - return null; - } - - @Override - public void copy(DataInputView source, DataOutputView target) throws IOException { - StringValue.copyString(source, target); - } - - @Override - public TypeSerializerSnapshot snapshotConfiguration() { - return null; - } -} diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer09.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer09.java index 1660063c4..b026bf2c6 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer09.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer09.java @@ -23,7 +23,17 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.MeterView; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; +import org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaMetricWrapper; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaDelegatePartitioner; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; + +import java.util.Map; import java.util.Properties; /** diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java index be9538464..05c934b27 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java @@ -25,13 +25,13 @@ import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.formats.json.JsonRowDeserializationSchema; -import org.apache.flink.formats.json.JsonRowSchemaConverter; +import org.apache.flink.formats.json.JsonSchemaConverter; import org.apache.flink.metrics.Counter; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ContainerNode; -import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; @@ -91,7 +91,7 @@ public CustomerJsonRowSerializationSchema(TypeInformation typeInfo) { * @see http://json-schema.org/ */ public CustomerJsonRowSerializationSchema(String jsonSchema) { - this(JsonRowSchemaConverter.convert(jsonSchema)); + this(JsonSchemaConverter.convert(jsonSchema)); } @Override diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafka09JsonTableSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafka09JsonTableSink.java index ffd44e9f7..508366435 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafka09JsonTableSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafka09JsonTableSink.java @@ -19,15 +19,14 @@ import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.functions.sink.SinkFunction; -import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducerBase; -import org.apache.flink.streaming.connectors.kafka.Kafka09TableSink; +import org.apache.flink.streaming.connectors.kafka.*; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaDelegatePartitioner; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.utils.TableConnectorUtils; +import org.apache.flink.streaming.connectors.kafka.partitioner.KafkaPartitioner; +import org.apache.flink.table.util.TableConnectorUtil; import org.apache.flink.types.Row; -import java.util.Optional; import java.util.Properties; /** @@ -37,34 +36,43 @@ * * @author maqi */ -public class CustomerKafka09JsonTableSink extends Kafka09TableSink { +public class CustomerKafka09JsonTableSink extends KafkaJsonTableSink { protected SerializationSchema schema; + public CustomerKafka09JsonTableSink(String topic, Properties properties, SerializationSchema schema) { + super(topic, properties, new FlinkFixedPartitioner<>()); + this.schema = schema; + } + public CustomerKafka09JsonTableSink(String topic, Properties properties, FlinkKafkaPartitioner partitioner, SerializationSchema schema) { + super(topic, properties, partitioner); + this.schema = schema; + } @Deprecated - public CustomerKafka09JsonTableSink(TableSchema schema, - String topic, - Properties properties, - Optional> partitioner, - SerializationSchema serializationSchema) { - super(schema, topic, properties, partitioner, serializationSchema); - this.schema = serializationSchema; + public CustomerKafka09JsonTableSink(String topic, Properties properties, KafkaPartitioner partitioner, SerializationSchema schema) { + super(topic, properties, new FlinkKafkaDelegatePartitioner<>(partitioner)); + this.schema = schema; + } + + @Override + protected FlinkKafkaProducerBase createKafkaProducer(String topic, Properties properties, SerializationSchema serializationSchema, FlinkKafkaPartitioner partitioner) { + return new CustomerFlinkKafkaProducer09(topic, serializationSchema, properties); } @Override - protected FlinkKafkaProducerBase createKafkaProducer(String s, Properties properties, SerializationSchema serializationSchema, Optional> optional) { - return new CustomerFlinkKafkaProducer09<>(topic, serializationSchema, properties); + protected Kafka09JsonTableSink createCopy() { + return new Kafka09JsonTableSink(topic, properties, partitioner); } @Override public void emitDataStream(DataStream dataStream) { - SinkFunction kafkaProducer = createKafkaProducer(topic, properties, schema, partitioner); + FlinkKafkaProducerBase kafkaProducer = createKafkaProducer(topic, properties, schema, partitioner); // always enable flush on checkpoint to achieve at-least-once if query runs with checkpointing enabled. - //kafkaProducer.setFlushOnCheckpoint(true); - dataStream.addSink(kafkaProducer).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); + kafkaProducer.setFlushOnCheckpoint(true); + dataStream.addSink(kafkaProducer).name(TableConnectorUtil.generateRuntimeName(this.getClass(), fieldNames)); } } diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index a6c3b98a4..864bf997f 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -23,19 +23,12 @@ import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.KafkaTableSinkBase; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.streaming.connectors.kafka.KafkaTableSink; +import org.apache.flink.table.sinks.AppendStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; - -import java.util.Optional; import java.util.Properties; /** @@ -45,7 +38,7 @@ * @author DocLi * @modifyer maqi */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { +public class KafkaSink implements AppendStreamTableSink, IStreamSinkGener { protected String[] fieldNames; @@ -58,28 +51,10 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< /** Serialization schema for encoding records to Kafka. */ protected SerializationSchema serializationSchema; - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - - protected int parallelism; - - - @Override public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka09SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafka09SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka09SinkTableInfo.getBootstrapServers()); - for (String key : kafka09SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka09SinkTableInfo.getKafkaParam(key)); - } - - this.partitioner = Optional.of(new FlinkFixedPartitioner<>()); this.fieldNames = kafka09SinkTableInfo.getFields(); TypeInformation[] types = new TypeInformation[kafka09SinkTableInfo.getFields().length]; for (int i = 0; i < kafka09SinkTableInfo.getFieldClasses().length; i++) { @@ -87,46 +62,30 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { } this.fieldTypes = types; - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - KafkaTableSinkBase kafkaTableSink = new CustomerKafka09JsonTableSink( - schema, + public void emitDataStream(DataStream dataStream) { + KafkaTableSink kafkaTableSink = new CustomerKafka09JsonTableSink( topic, properties, - partitioner, serializationSchema ); - DataStream ds = dataStream.map((Tuple2 record) -> { - return record.f1; - }).returns(getOutputType().getTypeAt(1)).setParallelism(parallelism); - - kafkaTableSink.emitDataStream(ds); + kafkaTableSink.emitDataStream(dataStream); } @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + public TypeInformation getOutputType() { + return new RowTypeInfo(fieldTypes, fieldNames); } @Override @@ -140,7 +99,7 @@ public TypeInformation[] getFieldTypes() { } @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + public TableSink configure(String[] fieldNames, TypeInformation[] fieldTypes) { this.fieldNames = fieldNames; this.fieldTypes = fieldTypes; return this; diff --git a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index dd4d4450f..3e1f707e2 100644 --- a/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka09/kafka09-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -42,10 +42,6 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map kafkaParam = new HashMap(); + public KafkaSinkTableInfo() { + super.setType(CURR_TYPE); + } public void addKafkaParam(String key,String value){ kafkaParam.put(key,value); diff --git a/kafka09/kafka09-source/pom.xml b/kafka09/kafka09-source/pom.xml index 84e88348a..1b008bd3e 100644 --- a/kafka09/kafka09-source/pom.xml +++ b/kafka09/kafka09-source/pom.xml @@ -14,6 +14,16 @@ kafka09-source http://maven.apache.org + + + + org.apache.flink + flink-connector-kafka-0.9_2.11 + ${flink.version} + + + + diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index 6353ea447..960842c53 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -23,17 +23,14 @@ import com.dtstack.flink.sql.source.AbsDeserialization; import com.dtstack.flink.sql.source.kafka.metric.KafkaTopicPartitionLagMetric; -import com.dtstack.flink.sql.table.TableInfo; -import com.fasterxml.jackson.databind.node.ArrayNode; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.base.Strings; +import org.apache.flink.calcite.shaded.com.google.common.base.Strings; import org.apache.flink.metrics.MetricGroup; -import com.google.common.collect.Maps; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.JsonNodeType; import org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread; import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; import org.apache.flink.types.Row; @@ -42,13 +39,10 @@ import org.apache.kafka.common.TopicPartition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.IOException; import java.lang.reflect.Field; -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Set; @@ -67,8 +61,6 @@ public class CustomerJsonDeserialization extends AbsDeserialization { private static final long serialVersionUID = 2385115520960444192L; - private static int dirtyDataFrequency = 1000; - private final ObjectMapper objectMapper = new ObjectMapper(); /** Type information describing the result type. */ @@ -80,6 +72,9 @@ public class CustomerJsonDeserialization extends AbsDeserialization { /** Types to parse fields as. Indices match fieldNames indices. */ private final TypeInformation[] fieldTypes; + /** Flag indicating whether to fail on a missing field. */ + private boolean failOnMissingField; + private AbstractFetcher fetcher; private boolean firstMsg = true; @@ -88,14 +83,15 @@ public class CustomerJsonDeserialization extends AbsDeserialization { private Map rowAndFieldMapping; - private List fieldExtraInfos; - public CustomerJsonDeserialization(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos){ + public CustomerJsonDeserialization(TypeInformation typeInfo, Map rowAndFieldMapping){ this.typeInfo = typeInfo; + this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); + this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); + this.rowAndFieldMapping= rowAndFieldMapping; - this.fieldExtraInfos = fieldExtraInfos; } @Override @@ -112,23 +108,18 @@ public Row deserialize(byte[] message) throws IOException { } try { - JsonNode root = objectMapper.readTree(message); - if (numInRecord.getCount() % dirtyDataFrequency == 0) { - LOG.info(root.toString()); - } - numInRecord.inc(); - numInBytes.inc(message.length); + if(message!=null){numInBytes.inc(message.length);} + JsonNode root = objectMapper.readTree(message); parseTree(root, null); Row row = new Row(fieldNames.length); for (int i = 0; i < fieldNames.length; i++) { JsonNode node = getIgnoreCase(fieldNames[i]); - TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); if (node == null) { - if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { + if (failOnMissingField) { throw new IllegalStateException("Failed to find field with name '" + fieldNames[i] + "'."); } else { @@ -136,19 +127,20 @@ public Row deserialize(byte[] message) throws IOException { } } else { // Read the value as specified type - Object value = convert(node, fieldTypes[i]); + Object value; + if (node.isValueNode()) { + value = objectMapper.treeToValue(node, fieldTypes[i].getTypeClass()); + } else { + value = node.toString(); + } row.setField(i, value); } } numInResolveRecord.inc(); return row; - } catch (Exception e) { + } catch (Throwable t) { //add metric of dirty data - if (dirtyDataCounter.getCount() % dirtyDataFrequency == 0) { - LOG.info("dirtyData: " + new String(message)); - LOG.info(" " ,e); - } dirtyDataCounter.inc(); return null; }finally { @@ -158,29 +150,33 @@ public Row deserialize(byte[] message) throws IOException { public JsonNode getIgnoreCase(String key) { String nodeMappingKey = rowAndFieldMapping.getOrDefault(key, key); + return nodeAndJsonNodeMapping.get(nodeMappingKey); } + public void setFailOnMissingField(boolean failOnMissingField) { + this.failOnMissingField = failOnMissingField; + } private void parseTree(JsonNode jsonNode, String prefix){ - if (jsonNode.isArray()) { - ArrayNode array = (ArrayNode) jsonNode; - for (int i = 0; i < array.size(); i++) { - JsonNode child = array.get(i); - String nodeKey = getNodeKey(prefix, i); - - if (child.isValueNode()) { - nodeAndJsonNodeMapping.put(nodeKey, child); - } else { - if (rowAndFieldMapping.containsValue(nodeKey)) { - nodeAndJsonNodeMapping.put(nodeKey, child); - } - parseTree(child, nodeKey); - } + if (jsonNode.isArray()) { + ArrayNode array = (ArrayNode) jsonNode; + for (int i = 0; i < array.size(); i++) { + JsonNode child = array.get(i); + String nodeKey = getNodeKey(prefix, i); + + if (child.isValueNode()) { + nodeAndJsonNodeMapping.put(nodeKey, child); + } else { + if (rowAndFieldMapping.containsValue(nodeKey)) { + nodeAndJsonNodeMapping.put(nodeKey, child); } - return; + parseTree(child, nodeKey); + } } + return; + } Iterator iterator = jsonNode.fieldNames(); while (iterator.hasNext()){ @@ -188,14 +184,15 @@ private void parseTree(JsonNode jsonNode, String prefix){ JsonNode child = jsonNode.get(next); String nodeKey = getNodeKey(prefix, next); - if (child.isValueNode()){ - nodeAndJsonNodeMapping.put(nodeKey, child); - } else if(child.isArray()){ - parseTree(child, nodeKey); - }else { - parseTree(child, nodeKey); - } + if (child.isValueNode()) { + nodeAndJsonNodeMapping.put(nodeKey, child); + } else { + if (rowAndFieldMapping.containsValue(nodeKey)) { + nodeAndJsonNodeMapping.put(nodeKey, child); + } + parseTree(child, nodeKey); } + } } private String getNodeKey(String prefix, String nodeName){ @@ -257,31 +254,4 @@ protected void registerPtMetric(AbstractFetcher fetcher) throws Exceptio private static String partitionLagMetricName(TopicPartition tp) { return tp + ".records-lag"; } - - private Object convert(JsonNode node, TypeInformation info) { - if (info.getTypeClass().equals(Types.BOOLEAN.getTypeClass())) { - return node.asBoolean(); - } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { - return node.asText(); - } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { - return Date.valueOf(node.asText()); - } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { - // local zone - return Time.valueOf(node.asText()); - } else if (info.getTypeClass().equals(Types.SQL_TIMESTAMP.getTypeClass())) { - // local zone - return Timestamp.valueOf(node.asText()); - } else { - // for types that were specified without JSON schema - // e.g. POJOs - try { - return objectMapper.treeToValue(node, info.getTypeClass()); - } catch (JsonProcessingException e) { - throw new IllegalStateException("Unsupported type information '" + info + "' for node: " + node); - } - } - } - - - } diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafka09Consumer.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafka09Consumer.java index 68d9c4958..09e684c75 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafka09Consumer.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafka09Consumer.java @@ -70,4 +70,4 @@ public void run(SourceContext sourceContext) throws Exception { customerJsonDeserialization.setFetcher(fetcher); return fetcher; } -} \ No newline at end of file +} diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 8de9ce703..326b2b7c1 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -16,7 +16,7 @@ * limitations under the License. */ - + package com.dtstack.flink.sql.source.kafka; @@ -27,9 +27,9 @@ import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer09; import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; @@ -61,7 +61,7 @@ public class KafkaSource implements IStreamSourceGener

{ */ @SuppressWarnings("rawtypes") @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { + public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { KafkaSourceTableInfo kafka09SourceTableInfo = (KafkaSourceTableInfo) sourceTableInfo; String topicName = kafka09SourceTableInfo.getTopic(); @@ -88,10 +88,10 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv FlinkKafkaConsumer09 kafkaSrc; if (BooleanUtils.isTrue(kafka09SourceTableInfo.getTopicIsPattern())) { kafkaSrc = new CustomerKafka09Consumer(Pattern.compile(topicName), - new CustomerJsonDeserialization(typeInformation, kafka09SourceTableInfo.getPhysicalFields(), kafka09SourceTableInfo.getFieldExtraInfoList()), props); + new CustomerJsonDeserialization(typeInformation, kafka09SourceTableInfo.getPhysicalFields()), props); } else { kafkaSrc = new CustomerKafka09Consumer(topicName, - new CustomerJsonDeserialization(typeInformation, kafka09SourceTableInfo.getPhysicalFields(), kafka09SourceTableInfo.getFieldExtraInfoList()), props); + new CustomerJsonDeserialization(typeInformation, kafka09SourceTableInfo.getPhysicalFields()), props); } //earliest,latest @@ -115,12 +115,6 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv String fields = StringUtils.join(kafka09SourceTableInfo.getFields(), ","); String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafka09SourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } - return tableEnv.fromDataStream(kafkaSource, fields); + return tableEnv.fromDataStream(env.addSource(kafkaSrc, sourceOperatorName, typeInformation), fields); } -} \ No newline at end of file +} diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 106d2aeb9..a0d71c1ac 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -21,11 +21,10 @@ package com.dtstack.flink.sql.source.kafka.table; import com.dtstack.flink.sql.table.AbsSourceParser; +import com.dtstack.flink.sql.table.SourceTableInfo; import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.MathUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Map; import java.util.regex.Matcher; @@ -40,11 +39,9 @@ public class KafkaSourceParser extends AbsSourceParser { - private static final Logger LOG = LoggerFactory.getLogger(KafkaSourceParser.class); - private static final String KAFKA_NEST_FIELD_KEY = "nestFieldKey"; - private static Pattern kafkaNestFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(\\w+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); + private static Pattern kafkaNestFieldKeyPattern = Pattern.compile("(?i)((\S+\.)*\S+)\s+(\w+)\s+AS\s+(\w+)$"); static { keyPatternMap.put(KAFKA_NEST_FIELD_KEY, kafkaNestFieldKeyPattern); @@ -57,18 +54,11 @@ static void dealNestField(Matcher matcher, TableInfo tableInfo) { String fieldType = matcher.group(3); String mappingField = matcher.group(4); Class fieldClass= ClassUtil.stringConvertClass(fieldType); - boolean notNull = matcher.group(5) != null; - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); - fieldExtraInfo.setNotNull(notNull); tableInfo.addPhysicalMappings(mappingField, physicalField); tableInfo.addField(mappingField); tableInfo.addFieldClass(fieldClass); tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(fieldExtraInfo); - if(LOG.isInfoEnabled()){ - LOG.info(physicalField + "--->" + mappingField + " Class: " + fieldClass.toString()); - } } @Override diff --git a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index e6098fb3c..9081b956b 100644 --- a/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka09/kafka09-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.source.kafka.table; import com.dtstack.flink.sql.table.SourceTableInfo; -import com.google.common.base.Preconditions; +import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; /** * Reason: diff --git a/kafka09/pom.xml b/kafka09/pom.xml index 86613de70..1ae761e9b 100644 --- a/kafka09/pom.xml +++ b/kafka09/pom.xml @@ -6,7 +6,7 @@ flink.sql com.dtstack.flink 1.0-SNAPSHOT - ../pom.xml + 4.0.0 diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerCsvSerialization.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerCsvSerialization.java deleted file mode 100644 index e39a2d3b6..000000000 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerCsvSerialization.java +++ /dev/null @@ -1,122 +0,0 @@ -package com.dtstack.flink.sql.sink.kafka; - -import org.apache.commons.lang3.StringEscapeUtils; -import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; -import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataOutputView; -import org.apache.flink.types.Row; -import org.apache.flink.types.StringValue; - -import java.io.IOException; - -import static org.apache.flink.api.java.typeutils.runtime.NullMaskUtils.writeNullMask; -/** - * - * Date: 2018/12/18 - * Company: www.dtstack.com - * - * @author DocLi - * - * @modifyer maqi - * - */ -public final class CustomerCsvSerialization extends TypeSerializerSingleton { - - private static final long serialVersionUID = 1L; - - private String fieldDelimiter = "\u0001"; - private TypeInformation[] fieldTypes; - private TypeSerializer[] fieldSerializers; - private static final Row EMPTY = null; - - public CustomerCsvSerialization(String fielddelimiter,TypeInformation[] fieldTypes) { - this.fieldDelimiter = fielddelimiter; - this.fieldTypes = fieldTypes; - this.fieldSerializers = (TypeSerializer[])createSerializer(new ExecutionConfig()); - } - - public TypeSerializer[] createSerializer(ExecutionConfig config) { - int len = fieldTypes.length; - TypeSerializer[] fieldSerializers = new TypeSerializer[len]; - for (int i = 0; i < len; i++) { - fieldSerializers[i] = fieldTypes[i].createSerializer(config); - } - return fieldSerializers; - } - - @Override - public boolean isImmutableType() { - return true; - } - - @Override - public Row createInstance() { - return EMPTY; - } - - @Override - public Row copy(Row from) { - return null; - } - - @Override - public Row copy(Row from, Row reuse) { - return null; - } - - @Override - public int getLength() { - return -1; - } - - @Override - public void serialize(Row record, DataOutputView target) throws IOException { - int len = fieldSerializers.length; - - if (record.getArity() != len) { - throw new RuntimeException("Row arity of from does not match serializers."); - } - - // write a null mask - writeNullMask(len, record, target); - - // serialize non-null fields - StringBuffer stringBuffer = new StringBuffer(); - for (int i = 0; i < len; i++) { - Object o = record.getField(i); - if (o != null) { - //fieldSerializers[i].serialize(o, target); - stringBuffer.append(o); - } - if(i != len-1){ - stringBuffer.append(StringEscapeUtils.unescapeJava(fieldDelimiter)); - //fieldSerializers[i].serialize(fieldDelimiter, target); - } - } - StringValue.writeString(stringBuffer.toString(), target); - } - - @Override - public Row deserialize(DataInputView source) throws IOException { - return null; - } - - @Override - public Row deserialize(Row reuse, DataInputView source) throws IOException { - return null; - } - - @Override - public void copy(DataInputView source, DataOutputView target) throws IOException { - StringValue.copyString(source, target); - } - - @Override - public TypeSerializerSnapshot snapshotConfiguration() { - return null; - } -} diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer010.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer010.java index 2f11c355d..50da17099 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer010.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer010.java @@ -23,7 +23,15 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.MeterView; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; +import org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaMetricWrapper; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaDelegatePartitioner; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; import java.util.Map; import java.util.Properties; diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java index 8033c851e..54c346fe5 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java @@ -25,13 +25,13 @@ import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.formats.json.JsonRowDeserializationSchema; -import org.apache.flink.formats.json.JsonRowSchemaConverter; +import org.apache.flink.formats.json.JsonSchemaConverter; import org.apache.flink.metrics.Counter; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ContainerNode; -import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; @@ -91,7 +91,7 @@ public CustomerJsonRowSerializationSchema(TypeInformation typeInfo) { * @see http://json-schema.org/ */ public CustomerJsonRowSerializationSchema(String jsonSchema) { - this(JsonRowSchemaConverter.convert(jsonSchema)); + this(JsonSchemaConverter.convert(jsonSchema)); } @Override diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafka10JsonTableSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafka10JsonTableSink.java index d3edc2d86..45dc6a331 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafka10JsonTableSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafka10JsonTableSink.java @@ -19,15 +19,16 @@ import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.functions.sink.SinkFunction; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducerBase; -import org.apache.flink.streaming.connectors.kafka.Kafka010TableSink; +import org.apache.flink.streaming.connectors.kafka.Kafka010JsonTableSink; +import org.apache.flink.streaming.connectors.kafka.KafkaJsonTableSink; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaDelegatePartitioner; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.utils.TableConnectorUtils; +import org.apache.flink.streaming.connectors.kafka.partitioner.KafkaPartitioner; +import org.apache.flink.table.util.TableConnectorUtil; import org.apache.flink.types.Row; -import java.util.Optional; import java.util.Properties; /** @@ -37,38 +38,43 @@ * * @author maqi */ -public class CustomerKafka10JsonTableSink extends Kafka010TableSink { +public class CustomerKafka10JsonTableSink extends KafkaJsonTableSink { protected SerializationSchema schema; - public CustomerKafka10JsonTableSink(TableSchema schema, - String topic, - Properties properties, - Optional> partitioner, - SerializationSchema serializationSchema) { - super(schema, topic, properties, partitioner, serializationSchema); - this.schema = serializationSchema; + public CustomerKafka10JsonTableSink(String topic, Properties properties, SerializationSchema schema) { + super(topic, properties, new FlinkFixedPartitioner<>()); + this.schema = schema; + } + + public CustomerKafka10JsonTableSink(String topic, Properties properties, FlinkKafkaPartitioner partitioner, SerializationSchema schema) { + super(topic, properties, partitioner); + this.schema = schema; } + @Deprecated + public CustomerKafka10JsonTableSink(String topic, Properties properties, KafkaPartitioner partitioner, SerializationSchema schema) { + super(topic, properties, new FlinkKafkaDelegatePartitioner<>(partitioner)); + this.schema = schema; + } @Override - protected FlinkKafkaProducerBase createKafkaProducer( - String topic, - Properties properties, - SerializationSchema serializationSchema, - Optional> partitioner) { + protected FlinkKafkaProducerBase createKafkaProducer(String topic, Properties properties, SerializationSchema serializationSchema, FlinkKafkaPartitioner partitioner) { return new CustomerFlinkKafkaProducer010(topic, serializationSchema, properties); } + @Override + protected Kafka010JsonTableSink createCopy() { + return new Kafka010JsonTableSink(topic, properties, partitioner); + } @Override public void emitDataStream(DataStream dataStream) { - SinkFunction kafkaProducer = createKafkaProducer(topic, properties, schema, partitioner); + FlinkKafkaProducerBase kafkaProducer = createKafkaProducer(topic, properties, schema, partitioner); // always enable flush on checkpoint to achieve at-least-once if query runs with checkpointing enabled. - //kafkaProducer.setFlushOnCheckpoint(true); - dataStream.addSink(kafkaProducer).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); - + kafkaProducer.setFlushOnCheckpoint(true); + dataStream.addSink(kafkaProducer).name(TableConnectorUtil.generateRuntimeName(this.getClass(), fieldNames)); } } diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 8c757ce4a..427bf14cc 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -23,15 +23,10 @@ import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.KafkaTableSinkBase; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.streaming.connectors.kafka.KafkaTableSink; +import org.apache.flink.table.sinks.AppendStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; @@ -47,7 +42,7 @@ * @modifyer maqi * */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { +public class KafkaSink implements AppendStreamTableSink, IStreamSinkGener { protected String[] fieldNames; @@ -58,30 +53,13 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener< protected Properties properties; - protected int parallelism; - /** Serialization schema for encoding records to Kafka. */ protected SerializationSchema serializationSchema; - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - @Override public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka10SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafka10SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka10SinkTableInfo.getBootstrapServers()); - - for (String key : kafka10SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka10SinkTableInfo.getKafkaParam(key)); - } - - this.partitioner = Optional.of(new FlinkFixedPartitioner<>()); this.fieldNames = kafka10SinkTableInfo.getFields(); TypeInformation[] types = new TypeInformation[kafka10SinkTableInfo.getFields().length]; for (int i = 0; i < kafka10SinkTableInfo.getFieldClasses().length; i++) { @@ -89,47 +67,30 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { } this.fieldTypes = types; - - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - KafkaTableSinkBase kafkaTableSink = new CustomerKafka10JsonTableSink( - schema, + public void emitDataStream(DataStream dataStream) { + KafkaTableSink kafkaTableSink = new CustomerKafka10JsonTableSink( topic, properties, - partitioner, serializationSchema ); - DataStream ds = dataStream.map((Tuple2 record) -> { - return record.f1; - }).returns(getOutputType().getTypeAt(1)).setParallelism(parallelism); - - kafkaTableSink.emitDataStream(ds); + kafkaTableSink.emitDataStream(dataStream); } @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + public TypeInformation getOutputType() { + return new RowTypeInfo(fieldTypes, fieldNames); } @Override @@ -143,7 +104,7 @@ public TypeInformation[] getFieldTypes() { } @Override - public TableSink> configure(String[] fieldNames, TypeInformation[] fieldTypes) { + public TableSink configure(String[] fieldNames, TypeInformation[] fieldTypes) { this.fieldNames = fieldNames; this.fieldTypes = fieldTypes; return this; diff --git a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index 073fe9111..9f250fc03 100644 --- a/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka10/kafka10-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -42,10 +42,6 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Mapsql.kafka10 com.dtstack.flink 1.0-SNAPSHOT - ../pom.xml 4.0.0 @@ -14,6 +13,17 @@ kafka10-source http://maven.apache.org + + + + org.apache.flink + flink-connector-kafka-0.10_2.11 + ${flink.version} + + + + + diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index 87482a392..f6099a278 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -23,17 +23,14 @@ import com.dtstack.flink.sql.source.AbsDeserialization; import com.dtstack.flink.sql.source.kafka.metric.KafkaTopicPartitionLagMetric; -import com.dtstack.flink.sql.table.TableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.base.Strings; +import org.apache.flink.calcite.shaded.com.google.common.base.Strings; import org.apache.flink.metrics.MetricGroup; -import com.google.common.collect.Maps; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.JsonNodeType; import org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread; import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; import org.apache.flink.types.Row; @@ -45,11 +42,7 @@ import java.io.IOException; import java.lang.reflect.Field; -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Set; @@ -68,8 +61,6 @@ public class CustomerJsonDeserialization extends AbsDeserialization { private static final long serialVersionUID = 2385115520960444192L; - private static int dirtyDataFrequency = 1000; - private final ObjectMapper objectMapper = new ObjectMapper(); /** Type information describing the result type. */ @@ -81,6 +72,9 @@ public class CustomerJsonDeserialization extends AbsDeserialization { /** Types to parse fields as. Indices match fieldNames indices. */ private final TypeInformation[] fieldTypes; + /** Flag indicating whether to fail on a missing field. */ + private boolean failOnMissingField; + private AbstractFetcher fetcher; private boolean firstMsg = true; @@ -89,14 +83,14 @@ public class CustomerJsonDeserialization extends AbsDeserialization { private Map rowAndFieldMapping; - private List fieldExtraInfos; - - public CustomerJsonDeserialization(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos){ + public CustomerJsonDeserialization(TypeInformation typeInfo, Map rowAndFieldMapping){ this.typeInfo = typeInfo; + this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); + this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); + this.rowAndFieldMapping= rowAndFieldMapping; - this.fieldExtraInfos = fieldExtraInfos; } @Override @@ -113,24 +107,18 @@ public Row deserialize(byte[] message) throws IOException { } try { - JsonNode root = objectMapper.readTree(message); - - if (numInRecord.getCount() % dirtyDataFrequency == 0) { - LOG.info(root.toString()); - } - numInRecord.inc(); - numInBytes.inc(message.length); + if(message!=null){numInBytes.inc(message.length);} + JsonNode root = objectMapper.readTree(message); parseTree(root, null); Row row = new Row(fieldNames.length); for (int i = 0; i < fieldNames.length; i++) { JsonNode node = getIgnoreCase(fieldNames[i]); - TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); if (node == null) { - if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { + if (failOnMissingField) { throw new IllegalStateException("Failed to find field with name '" + fieldNames[i] + "'."); } else { @@ -138,19 +126,20 @@ public Row deserialize(byte[] message) throws IOException { } } else { // Read the value as specified type - Object value = convert(node, fieldTypes[i]); + Object value; + if (node.isValueNode()) { + value = objectMapper.treeToValue(node, fieldTypes[i].getTypeClass()); + } else { + value = node.toString(); + } row.setField(i, value); } } numInResolveRecord.inc(); return row; - } catch (Exception e) { + } catch (Throwable t) { //add metric of dirty data - if (dirtyDataCounter.getCount() % dirtyDataFrequency == 0 || LOG.isDebugEnabled()) { - LOG.info("dirtyData: " + new String(message)); - LOG.error(" ", e); - } dirtyDataCounter.inc(); return null; }finally { @@ -164,25 +153,30 @@ public JsonNode getIgnoreCase(String key) { return nodeAndJsonNodeMapping.get(nodeMappingKey); } - private void parseTree(JsonNode jsonNode, String prefix){ - if (jsonNode.isArray()) { - ArrayNode array = (ArrayNode) jsonNode; - for (int i = 0; i < array.size(); i++) { - JsonNode child = array.get(i); - String nodeKey = getNodeKey(prefix, i); + public void setFailOnMissingField(boolean failOnMissingField) { + this.failOnMissingField = failOnMissingField; + } - if (child.isValueNode()) { - nodeAndJsonNodeMapping.put(nodeKey, child); - } else { - if (rowAndFieldMapping.containsValue(nodeKey)) { - nodeAndJsonNodeMapping.put(nodeKey, child); - } - parseTree(child, nodeKey); - } + private void parseTree(JsonNode jsonNode, String prefix){ + + if (jsonNode.isArray()) { + ArrayNode array = (ArrayNode) jsonNode; + for (int i = 0; i < array.size(); i++) { + JsonNode child = array.get(i); + String nodeKey = getNodeKey(prefix, i); + + if (child.isValueNode()) { + nodeAndJsonNodeMapping.put(nodeKey, child); + } else { + if (rowAndFieldMapping.containsValue(nodeKey)) { + nodeAndJsonNodeMapping.put(nodeKey, child); } - return; + parseTree(child, nodeKey); + } } + return; + } Iterator iterator = jsonNode.fieldNames(); while (iterator.hasNext()){ @@ -190,14 +184,15 @@ private void parseTree(JsonNode jsonNode, String prefix){ JsonNode child = jsonNode.get(next); String nodeKey = getNodeKey(prefix, next); - if (child.isValueNode()) { - nodeAndJsonNodeMapping.put(nodeKey, child); - } else if(child.isArray()){ - parseTree(child, nodeKey); - } else { - parseTree(child, nodeKey); - } + if (child.isValueNode()) { + nodeAndJsonNodeMapping.put(nodeKey, child); + } else { + if (rowAndFieldMapping.containsValue(nodeKey)) { + nodeAndJsonNodeMapping.put(nodeKey, child); + } + parseTree(child, nodeKey); } + } } private String getNodeKey(String prefix, String nodeName){ @@ -259,28 +254,4 @@ protected void registerPtMetric(AbstractFetcher fetcher) throws Exceptio private static String partitionLagMetricName(TopicPartition tp) { return tp + ".records-lag"; } - - private Object convert(JsonNode node, TypeInformation info) { - if (info.getTypeClass().equals(Types.BOOLEAN.getTypeClass())) { - return node.asBoolean(); - } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { - return node.asText(); - } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { - return Date.valueOf(node.asText()); - } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { - // local zone - return Time.valueOf(node.asText()); - } else if (info.getTypeClass().equals(Types.SQL_TIMESTAMP.getTypeClass())) { - // local zone - return Timestamp.valueOf(node.asText()); - } else { - // for types that were specified without JSON schema - // e.g. POJOs - try { - return objectMapper.treeToValue(node, info.getTypeClass()); - } catch (JsonProcessingException e) { - throw new IllegalStateException("Unsupported type information '" + info + "' for node: " + node); - } - } - } } diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafka010Consumer.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafka010Consumer.java index 945c47407..f6878473b 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafka010Consumer.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafka010Consumer.java @@ -71,4 +71,4 @@ public void run(SourceContext sourceContext) throws Exception { return fetcher; } -} \ No newline at end of file +} diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index ab5ad8833..052dd5339 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -28,7 +28,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; @@ -90,10 +89,10 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv FlinkKafkaConsumer010 kafkaSrc; if (BooleanUtils.isTrue(kafka010SourceTableInfo.getTopicIsPattern())) { kafkaSrc = new CustomerKafka010Consumer(Pattern.compile(topicName), - new CustomerJsonDeserialization(typeInformation, kafka010SourceTableInfo.getPhysicalFields(), kafka010SourceTableInfo.getFieldExtraInfoList()), props); + new CustomerJsonDeserialization(typeInformation, kafka010SourceTableInfo.getPhysicalFields()), props); } else { kafkaSrc = new CustomerKafka010Consumer(topicName, - new CustomerJsonDeserialization(typeInformation, kafka010SourceTableInfo.getPhysicalFields(), kafka010SourceTableInfo.getFieldExtraInfoList()), props); + new CustomerJsonDeserialization(typeInformation, kafka010SourceTableInfo.getPhysicalFields()), props); } //earliest,latest @@ -117,12 +116,6 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv String fields = StringUtils.join(kafka010SourceTableInfo.getFields(), ","); String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafka010SourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } - return tableEnv.fromDataStream(kafkaSource, fields); + return tableEnv.fromDataStream(env.addSource(kafkaSrc, sourceOperatorName, typeInformation), fields); } -} \ No newline at end of file +} diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 563110726..9604b618d 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -24,8 +24,6 @@ import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.MathUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Map; import java.util.regex.Matcher; @@ -40,11 +38,9 @@ public class KafkaSourceParser extends AbsSourceParser { - private static final Logger LOG = LoggerFactory.getLogger(KafkaSourceParser.class); - private static final String KAFKA_NEST_FIELD_KEY = "nestFieldKey"; - private static Pattern kafkaNestFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(\\w+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); + private static Pattern kafkaNestFieldKeyPattern = Pattern.compile("(?i)((\S+\.)*\S+)\s+(\w+)\s+AS\s+(\w+)$"); static { keyPatternMap.put(KAFKA_NEST_FIELD_KEY, kafkaNestFieldKeyPattern); @@ -62,18 +58,11 @@ static void dealNestField(Matcher matcher, TableInfo tableInfo) { String fieldType = matcher.group(3); String mappingField = matcher.group(4); Class fieldClass= ClassUtil.stringConvertClass(fieldType); - boolean notNull = matcher.group(5) != null; - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); - fieldExtraInfo.setNotNull(notNull); tableInfo.addPhysicalMappings(mappingField, physicalField); tableInfo.addField(mappingField); tableInfo.addFieldClass(fieldClass); tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(fieldExtraInfo); - if(LOG.isInfoEnabled()){ - LOG.info(physicalField + "--->" + mappingField + " Class: " + fieldClass.toString()); - } } @Override diff --git a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index da78a2af6..113159450 100644 --- a/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka10/kafka10-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -21,7 +21,7 @@ package com.dtstack.flink.sql.source.kafka.table; import com.dtstack.flink.sql.table.SourceTableInfo; -import com.google.common.base.Preconditions; +import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; /** * Reason: diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer011.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer011.java index 6d90d44c9..fba215a39 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer011.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerFlinkKafkaProducer011.java @@ -23,8 +23,17 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.MeterView; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; +import org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaMetricWrapper; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaDelegatePartitioner; +import org.apache.kafka.clients.producer.Callback; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import java.util.Map; import java.util.Properties; /** @@ -56,4 +65,4 @@ public void open(Configuration configuration) { super.open(configuration); } -} \ No newline at end of file +} diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java index 0d575a5fd..487d938ce 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerJsonRowSerializationSchema.java @@ -17,25 +17,29 @@ package com.dtstack.flink.sql.sink.kafka; +import com.dtstack.flink.sql.sink.MetricOutputFormat; import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo; import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo; import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.configuration.Configuration; import org.apache.flink.formats.json.JsonRowDeserializationSchema; -import org.apache.flink.formats.json.JsonRowSchemaConverter; +import org.apache.flink.formats.json.JsonSchemaConverter; import org.apache.flink.metrics.Counter; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ContainerNode; -import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ContainerNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; +import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Time; @@ -93,7 +97,7 @@ public CustomerJsonRowSerializationSchema(TypeInformation typeInfo) { * @see http://json-schema.org/ */ public CustomerJsonRowSerializationSchema(String jsonSchema) { - this(JsonRowSchemaConverter.convert(jsonSchema)); + this(JsonSchemaConverter.convert(jsonSchema)); } @Override @@ -108,7 +112,7 @@ public byte[] serialize(Row row) { return mapper.writeValueAsBytes(node); } catch (Throwable t) { throw new RuntimeException("Could not serialize row '" + row + "'. " + - "Make sure that the schema matches the input.", t); + "Make sure that the schema matches the input.", t); } } @@ -124,7 +128,7 @@ private ObjectNode convertRow(ObjectNode reuse, RowTypeInfo info, Row row) { // validate the row if (row.getArity() != fieldNames.length) { throw new IllegalStateException(String.format( - "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); + "Number of elements in the row '%s' is different from number of field names: %d", row, fieldNames.length)); } for (int i = 0; i < fieldNames.length; i++) { @@ -218,4 +222,4 @@ public Counter getCounter() { public void setCounter(Counter counter) { this.counter = counter; } -} \ No newline at end of file +} diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafka11JsonTableSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafka11JsonTableSink.java index 89d1543a5..d9d122678 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafka11JsonTableSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/CustomerKafka11JsonTableSink.java @@ -19,14 +19,16 @@ import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.functions.sink.SinkFunction; -import org.apache.flink.streaming.connectors.kafka.Kafka011TableSink; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducerBase; +import org.apache.flink.streaming.connectors.kafka.Kafka010JsonTableSink; +import org.apache.flink.streaming.connectors.kafka.KafkaJsonTableSink; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; +import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaDelegatePartitioner; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.utils.TableConnectorUtils; +import org.apache.flink.streaming.connectors.kafka.partitioner.KafkaPartitioner; +import org.apache.flink.table.util.TableConnectorUtil; import org.apache.flink.types.Row; -import java.util.Optional; import java.util.Properties; /** @@ -37,31 +39,43 @@ * * @author maqi */ -public class CustomerKafka11JsonTableSink extends Kafka011TableSink { +public class CustomerKafka11JsonTableSink extends KafkaJsonTableSink { - protected SerializationSchema schema; + protected SerializationSchema schema; - public CustomerKafka11JsonTableSink(TableSchema schema, - String topic, - Properties properties, - Optional> partitioner, - SerializationSchema serializationSchema) { + public CustomerKafka11JsonTableSink(String topic, Properties properties, SerializationSchema schema) { + super(topic, properties, new FlinkFixedPartitioner<>()); + this.schema = schema; + } - super(schema, topic, properties, partitioner, serializationSchema); - this.schema = serializationSchema; + public CustomerKafka11JsonTableSink(String topic, Properties properties, FlinkKafkaPartitioner partitioner, SerializationSchema schema) { + super(topic, properties, partitioner); + this.schema = schema; } + + @Deprecated + public CustomerKafka11JsonTableSink(String topic, Properties properties, KafkaPartitioner partitioner, SerializationSchema schema) { + super(topic, properties, new FlinkKafkaDelegatePartitioner<>(partitioner)); + this.schema = schema; + } + //TODO 暂时使用010 @Override - protected SinkFunction createKafkaProducer(String topic, Properties properties, SerializationSchema serializationSchema, Optional> optional) { + protected FlinkKafkaProducerBase createKafkaProducer(String topic, Properties properties, SerializationSchema serializationSchema, FlinkKafkaPartitioner partitioner) { return new CustomerFlinkKafkaProducer011(topic, serializationSchema, properties); } + @Override + protected Kafka010JsonTableSink createCopy() { + return new Kafka010JsonTableSink(topic, properties, partitioner); + } + @Override public void emitDataStream(DataStream dataStream) { - SinkFunction kafkaProducer = createKafkaProducer(topic, properties, schema, partitioner); + FlinkKafkaProducerBase kafkaProducer = createKafkaProducer(topic, properties, schema, partitioner); // always enable flush on checkpoint to achieve at-least-once if query runs with checkpointing enabled. - //kafkaProducer.setFlushOnCheckpoint(true); - dataStream.addSink(kafkaProducer).name(TableConnectorUtils.generateRuntimeName(this.getClass(), getFieldNames())); + kafkaProducer.setFlushOnCheckpoint(true); + dataStream.addSink(kafkaProducer).name(TableConnectorUtil.generateRuntimeName(this.getClass(), fieldNames)); } -} \ No newline at end of file +} diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java index 5b0ac683f..d4cf64e9b 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/KafkaSink.java @@ -23,19 +23,13 @@ import com.dtstack.flink.sql.table.TargetTableInfo; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.api.java.typeutils.TupleTypeInfo; +import org.apache.flink.formats.json.JsonRowSerializationSchema; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.connectors.kafka.KafkaTableSinkBase; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner; -import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.sinks.RetractStreamTableSink; +import org.apache.flink.streaming.connectors.kafka.KafkaTableSink; +import org.apache.flink.table.sinks.AppendStreamTableSink; import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; - -import java.util.Optional; import java.util.Properties; /** @@ -48,7 +42,7 @@ * @modifyer maqi * */ -public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener { +public class KafkaSink implements AppendStreamTableSink, IStreamSinkGener { protected String[] fieldNames; @@ -56,32 +50,15 @@ public class KafkaSink implements RetractStreamTableSink, IStreamSinkGener protected String topic; - protected int parallelism; - protected Properties properties; /** Serialization schema for encoding records to Kafka. */ protected SerializationSchema serializationSchema; - /** The schema of the table. */ - private TableSchema schema; - - /** Partitioner to select Kafka partition for each item. */ - protected Optional> partitioner; - - @Override public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { KafkaSinkTableInfo kafka11SinkTableInfo = (KafkaSinkTableInfo) targetTableInfo; this.topic = kafka11SinkTableInfo.getTopic(); - - properties = new Properties(); - properties.setProperty("bootstrap.servers", kafka11SinkTableInfo.getBootstrapServers()); - - for (String key : kafka11SinkTableInfo.getKafkaParamKeys()) { - properties.setProperty(key, kafka11SinkTableInfo.getKafkaParam(key)); - } - this.partitioner = Optional.of(new FlinkFixedPartitioner<>()); this.fieldNames = kafka11SinkTableInfo.getFields(); TypeInformation[] types = new TypeInformation[kafka11SinkTableInfo.getFields().length]; for (int i = 0; i < kafka11SinkTableInfo.getFieldClasses().length; i++) { @@ -89,46 +66,29 @@ public KafkaSink genStreamSink(TargetTableInfo targetTableInfo) { } this.fieldTypes = types; - TableSchema.Builder schemaBuilder = TableSchema.builder(); - for (int i=0;i getRecordType() { - return new RowTypeInfo(fieldTypes, fieldNames); - } - - @Override - public void emitDataStream(DataStream> dataStream) { - KafkaTableSinkBase kafkaTableSink = new CustomerKafka11JsonTableSink( - schema, + public void emitDataStream(DataStream dataStream) { + KafkaTableSink kafkaTableSink = new CustomerKafka11JsonTableSink( topic, properties, - partitioner, serializationSchema ); - DataStream ds = dataStream.map((Tuple2 record) -> { - return record.f1; - }).returns(getOutputType().getTypeAt(1)).setParallelism(parallelism); - - kafkaTableSink.emitDataStream(ds); + kafkaTableSink.emitDataStream(dataStream); } @Override - public TupleTypeInfo> getOutputType() { - return new TupleTypeInfo(org.apache.flink.table.api.Types.BOOLEAN(), new RowTypeInfo(fieldTypes, fieldNames)); + public TypeInformation getOutputType() { + return new RowTypeInfo(fieldTypes, fieldNames); } @Override @@ -142,7 +102,7 @@ public TypeInformation[] getFieldTypes() { } @Override - public TableSink>configure(String[] fieldNames, TypeInformation[] fieldTypes) { + public TableSink configure(String[] fieldNames, TypeInformation[] fieldTypes) { this.fieldNames = fieldNames; this.fieldTypes = fieldTypes; return this; diff --git a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java index ca4ffd5cf..4ee9d99f0 100644 --- a/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java +++ b/kafka11/kafka11-sink/src/main/java/com/dtstack/flink/sql/sink/kafka/table/KafkaSinkParser.java @@ -38,29 +38,16 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map kafkaParam = new HashMap(); - private String topic; + public Map kafkaParam = new HashMap(); + + public KafkaSinkTableInfo() { + super.setType(CURR_TYPE); + } public void addKafkaParam(String key,String value){ kafkaParam.put(key,value); @@ -64,6 +63,7 @@ public Set getKafkaParamKeys(){ } + public String getBootstrapServers() { return bootstrapServers; } @@ -80,17 +80,16 @@ public void setTopic(String topic) { this.topic = topic; } - @Override public boolean check() { Preconditions.checkNotNull(bootstrapServers, "kafka of bootstrapServers is required"); Preconditions.checkNotNull(topic, "kafka of topic is required"); - //Preconditions.checkNotNull(kafkaParam.get("groupId"), "kafka of groupId is required"); return false; } @Override public String getType() { +// return super.getType() + SOURCE_SUFFIX; return super.getType(); } -} \ No newline at end of file +} diff --git a/kafka11/kafka11-source/pom.xml b/kafka11/kafka11-source/pom.xml index 90a6de20b..130e66344 100644 --- a/kafka11/kafka11-source/pom.xml +++ b/kafka11/kafka11-source/pom.xml @@ -4,7 +4,6 @@ sql.kafka11 com.dtstack.flink 1.0-SNAPSHOT - ../pom.xml 4.0.0 @@ -14,6 +13,15 @@ kafka11-source http://maven.apache.org + + + + org.apache.flink + flink-connector-kafka-0.11_2.11 + ${flink.version} + + + diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java index b433804af..cc3d57a80 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerJsonDeserialization.java @@ -23,17 +23,14 @@ import com.dtstack.flink.sql.source.AbsDeserialization; import com.dtstack.flink.sql.source.kafka.metric.KafkaTopicPartitionLagMetric; -import com.dtstack.flink.sql.table.TableInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.base.Strings; +import org.apache.flink.calcite.shaded.com.google.common.base.Strings; import org.apache.flink.metrics.MetricGroup; -import com.google.common.collect.Maps; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.JsonNodeType; import org.apache.flink.streaming.connectors.kafka.internal.KafkaConsumerThread; import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; import org.apache.flink.types.Row; @@ -45,11 +42,7 @@ import java.io.IOException; import java.lang.reflect.Field; -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Set; @@ -70,8 +63,6 @@ public class CustomerJsonDeserialization extends AbsDeserialization { private static final long serialVersionUID = 2385115520960444192L; - private static int dirtyDataFrequency = 1000; - private final ObjectMapper objectMapper = new ObjectMapper(); /** Type information describing the result type. */ @@ -83,6 +74,9 @@ public class CustomerJsonDeserialization extends AbsDeserialization { /** Types to parse fields as. Indices match fieldNames indices. */ private final TypeInformation[] fieldTypes; + /** Flag indicating whether to fail on a missing field. */ + private boolean failOnMissingField; + private AbstractFetcher fetcher; private boolean firstMsg = true; @@ -91,14 +85,15 @@ public class CustomerJsonDeserialization extends AbsDeserialization { private Map rowAndFieldMapping; - private List fieldExtraInfos; - public CustomerJsonDeserialization(TypeInformation typeInfo, Map rowAndFieldMapping, List fieldExtraInfos){ + public CustomerJsonDeserialization(TypeInformation typeInfo, Map rowAndFieldMapping){ this.typeInfo = typeInfo; + this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); + this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); + this.rowAndFieldMapping= rowAndFieldMapping; - this.fieldExtraInfos = fieldExtraInfos; } @Override @@ -115,24 +110,18 @@ public Row deserialize(byte[] message) throws IOException { } try { - JsonNode root = objectMapper.readTree(message); - - if (numInRecord.getCount() % dirtyDataFrequency == 0) { - LOG.info(root.toString()); - } - numInRecord.inc(); - numInBytes.inc(message.length); + if(message!=null){numInBytes.inc(message.length);} + JsonNode root = objectMapper.readTree(message); parseTree(root, null); Row row = new Row(fieldNames.length); for (int i = 0; i < fieldNames.length; i++) { JsonNode node = getIgnoreCase(fieldNames[i]); - TableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i); if (node == null) { - if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) { + if (failOnMissingField) { throw new IllegalStateException("Failed to find field with name '" + fieldNames[i] + "'."); } else { @@ -140,20 +129,20 @@ public Row deserialize(byte[] message) throws IOException { } } else { // Read the value as specified type - - Object value = convert(node, fieldTypes[i]); + Object value; + if (node.isValueNode()) { + value = objectMapper.treeToValue(node, fieldTypes[i].getTypeClass()); + } else { + value = node.toString(); + } row.setField(i, value); } } numInResolveRecord.inc(); return row; - } catch (Exception e) { + } catch (Throwable t) { //add metric of dirty data - if (dirtyDataCounter.getCount() % dirtyDataFrequency == 0) { - LOG.info("dirtyData: " + new String(message)); - LOG.error("" , e); - } dirtyDataCounter.inc(); return null; }finally { @@ -163,43 +152,49 @@ public Row deserialize(byte[] message) throws IOException { public JsonNode getIgnoreCase(String key) { String nodeMappingKey = rowAndFieldMapping.getOrDefault(key, key); + return nodeAndJsonNodeMapping.get(nodeMappingKey); } - private void parseTree(JsonNode jsonNode, String prefix){ + public void setFailOnMissingField(boolean failOnMissingField) { + this.failOnMissingField = failOnMissingField; + } - if (jsonNode.isArray()) { - ArrayNode array = (ArrayNode) jsonNode; - for (int i = 0; i < array.size(); i++) { - JsonNode child = array.get(i); - String nodeKey = getNodeKey(prefix, i); + private void parseTree(JsonNode jsonNode, String prefix){ - if (child.isValueNode()) { - nodeAndJsonNodeMapping.put(nodeKey, child); - } else { - if (rowAndFieldMapping.containsValue(nodeKey)) { - nodeAndJsonNodeMapping.put(nodeKey, child); - } - parseTree(child, nodeKey); - } + if (jsonNode.isArray()) { + ArrayNode array = (ArrayNode) jsonNode; + for (int i = 0; i < array.size(); i++) { + JsonNode child = array.get(i); + String nodeKey = getNodeKey(prefix, i); + + if (child.isValueNode()) { + nodeAndJsonNodeMapping.put(nodeKey, child); + } else { + if (rowAndFieldMapping.containsValue(nodeKey)) { + nodeAndJsonNodeMapping.put(nodeKey, child); } - return; + parseTree(child, nodeKey); + } } + return; + } - Iterator iterator = jsonNode.fieldNames(); - while (iterator.hasNext()){ - String next = iterator.next(); - JsonNode child = jsonNode.get(next); - String nodeKey = getNodeKey(prefix, next); - - if (child.isValueNode()){ - nodeAndJsonNodeMapping.put(nodeKey, child); - }else if(child.isArray()){ - parseTree(child, nodeKey); - }else { - parseTree(child, nodeKey); - } + Iterator iterator = jsonNode.fieldNames(); + while (iterator.hasNext()) { + String next = iterator.next(); + JsonNode child = jsonNode.get(next); + String nodeKey = getNodeKey(prefix, next); + + if (child.isValueNode()) { + nodeAndJsonNodeMapping.put(nodeKey, child); + } else { + if (rowAndFieldMapping.containsValue(nodeKey)) { + nodeAndJsonNodeMapping.put(nodeKey, child); + } + parseTree(child, nodeKey); } + } } private String getNodeKey(String prefix, String nodeName){ @@ -261,29 +256,4 @@ protected void registerPtMetric(AbstractFetcher fetcher) throws Exceptio private static String partitionLagMetricName(TopicPartition tp) { return tp + ".records-lag"; } - - private Object convert(JsonNode node, TypeInformation info) { - if (info.getTypeClass().equals(Types.BOOLEAN.getTypeClass())) { - return node.asBoolean(); - } else if (info.getTypeClass().equals(Types.STRING.getTypeClass())) { - return node.asText(); - } else if (info.getTypeClass().equals(Types.SQL_DATE.getTypeClass())) { - return Date.valueOf(node.asText()); - } else if (info.getTypeClass().equals(Types.SQL_TIME.getTypeClass())) { - // local zone - return Time.valueOf(node.asText()); - } else if (info.getTypeClass().equals(Types.SQL_TIMESTAMP.getTypeClass())) { - // local zone - return Timestamp.valueOf(node.asText()); - } else { - // for types that were specified without JSON schema - // e.g. POJOs - try { - return objectMapper.treeToValue(node, info.getTypeClass()); - } catch (JsonProcessingException e) { - throw new IllegalStateException("Unsupported type information '" + info + "' for node: " + node); - } - } - } - } diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafka011Consumer.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafka011Consumer.java index 0a6c7cfca..7cc1f6f40 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafka011Consumer.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/CustomerKafka011Consumer.java @@ -75,4 +75,4 @@ public void run(SourceContext sourceContext) throws Exception { customerJsonDeserialization.setFetcher(fetcher); return fetcher; } -} \ No newline at end of file +} diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java index 8a552cc40..54e4190ac 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/KafkaSource.java @@ -28,7 +28,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; @@ -90,12 +89,13 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv FlinkKafkaConsumer011 kafkaSrc; if (BooleanUtils.isTrue(kafka011SourceTableInfo.getTopicIsPattern())) { kafkaSrc = new CustomerKafka011Consumer(Pattern.compile(topicName), - new CustomerJsonDeserialization(typeInformation, kafka011SourceTableInfo.getPhysicalFields(), kafka011SourceTableInfo.getFieldExtraInfoList()), props); + new CustomerJsonDeserialization(typeInformation, kafka011SourceTableInfo.getPhysicalFields()), props); } else { kafkaSrc = new CustomerKafka011Consumer(topicName, - new CustomerJsonDeserialization(typeInformation, kafka011SourceTableInfo.getPhysicalFields(), kafka011SourceTableInfo.getFieldExtraInfoList()), props); + new CustomerJsonDeserialization(typeInformation, kafka011SourceTableInfo.getPhysicalFields()), props); } + //earliest,latest if ("earliest".equalsIgnoreCase(kafka011SourceTableInfo.getOffsetReset())) { kafkaSrc.setStartFromEarliest(); @@ -117,12 +117,6 @@ public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnv String fields = StringUtils.join(kafka011SourceTableInfo.getFields(), ","); String sourceOperatorName = SOURCE_OPERATOR_NAME_TPL.replace("${topic}", topicName).replace("${table}", sourceTableInfo.getName()); - - DataStreamSource kafkaSource = env.addSource(kafkaSrc, sourceOperatorName, typeInformation); - Integer parallelism = kafka011SourceTableInfo.getParallelism(); - if (parallelism != null) { - kafkaSource.setParallelism(parallelism); - } - return tableEnv.fromDataStream(kafkaSource, fields); + return tableEnv.fromDataStream(env.addSource(kafkaSrc, sourceOperatorName, typeInformation), fields); } } diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java index 07e12b0c0..408981e53 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceParser.java @@ -24,8 +24,6 @@ import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.ClassUtil; import com.dtstack.flink.sql.util.MathUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Map; import java.util.regex.Matcher; @@ -40,11 +38,9 @@ public class KafkaSourceParser extends AbsSourceParser { - private static final Logger LOG = LoggerFactory.getLogger(KafkaSourceParser.class); - private static final String KAFKA_NEST_FIELD_KEY = "nestFieldKey"; - private static Pattern kafkaNestFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(\\w+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$"); + private static Pattern kafkaNestFieldKeyPattern = Pattern.compile("(?i)((\S+\.)*\S+)\s+(\w+)\s+AS\s+(\w+)$"); static { keyPatternMap.put(KAFKA_NEST_FIELD_KEY, kafkaNestFieldKeyPattern); @@ -62,18 +58,11 @@ static void dealNestField(Matcher matcher, TableInfo tableInfo) { String fieldType = matcher.group(3); String mappingField = matcher.group(4); Class fieldClass= ClassUtil.stringConvertClass(fieldType); - boolean notNull = matcher.group(5) != null; - TableInfo.FieldExtraInfo fieldExtraInfo = new TableInfo.FieldExtraInfo(); - fieldExtraInfo.setNotNull(notNull); tableInfo.addPhysicalMappings(mappingField, physicalField); tableInfo.addField(mappingField); tableInfo.addFieldClass(fieldClass); tableInfo.addFieldType(fieldType); - tableInfo.addFieldExtraInfo(fieldExtraInfo); - if(LOG.isInfoEnabled()){ - LOG.info(physicalField + "--->" + mappingField + " Class: " + fieldClass.toString()); - } } @Override diff --git a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java index 7a166695b..97b360573 100644 --- a/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java +++ b/kafka11/kafka11-source/src/main/java/com/dtstack/flink/sql/source/kafka/table/KafkaSourceTableInfo.java @@ -20,7 +20,8 @@ package com.dtstack.flink.sql.source.kafka.table; import com.dtstack.flink.sql.table.SourceTableInfo; -import com.google.common.base.Preconditions; +import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; +import org.apache.flink.table.shaded.org.apache.commons.lang.BooleanUtils; /** * Reason: diff --git a/kudu/kudu-side/kudu-all-side/pom.xml b/kudu/kudu-side/kudu-all-side/pom.xml index 5c35d4cd9..7629d0c75 100644 --- a/kudu/kudu-side/kudu-all-side/pom.xml +++ b/kudu/kudu-side/kudu-all-side/pom.xml @@ -39,7 +39,7 @@ - org.slf4j + diff --git a/kudu/kudu-side/kudu-async-side/pom.xml b/kudu/kudu-side/kudu-async-side/pom.xml index 6ab098ea9..7e2d96396 100644 --- a/kudu/kudu-side/kudu-async-side/pom.xml +++ b/kudu/kudu-side/kudu-async-side/pom.xml @@ -57,7 +57,7 @@ - org.slf4j + diff --git a/kudu/kudu-sink/pom.xml b/kudu/kudu-sink/pom.xml index 9c1bb2a6d..3225352ec 100644 --- a/kudu/kudu-sink/pom.xml +++ b/kudu/kudu-sink/pom.xml @@ -32,7 +32,7 @@ - org.slf4j + diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java b/kudu/kudu-sink/src/main/java/com.dtstack.flink.sql.sink.kudu/KuduOutputFormat.java similarity index 100% rename from kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduOutputFormat.java rename to kudu/kudu-sink/src/main/java/com.dtstack.flink.sql.sink.kudu/KuduOutputFormat.java diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java b/kudu/kudu-sink/src/main/java/com.dtstack.flink.sql.sink.kudu/KuduSink.java similarity index 100% rename from kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/KuduSink.java rename to kudu/kudu-sink/src/main/java/com.dtstack.flink.sql.sink.kudu/KuduSink.java diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java b/kudu/kudu-sink/src/main/java/com.dtstack.flink.sql.sink.kudu/table/KuduSinkParser.java similarity index 100% rename from kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduSinkParser.java rename to kudu/kudu-sink/src/main/java/com.dtstack.flink.sql.sink.kudu/table/KuduSinkParser.java diff --git a/kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java b/kudu/kudu-sink/src/main/java/com.dtstack.flink.sql.sink.kudu/table/KuduTableInfo.java similarity index 100% rename from kudu/kudu-sink/src/main/java/com/dtstack/flink/sql/sink/kudu/table/KuduTableInfo.java rename to kudu/kudu-sink/src/main/java/com.dtstack.flink.sql.sink.kudu/table/KuduTableInfo.java diff --git a/launcher/job/kafkaNotNull.txt b/launcher/job/kafkaNotNull.txt deleted file mode 100644 index 0515f2036..000000000 --- a/launcher/job/kafkaNotNull.txt +++ /dev/null @@ -1,34 +0,0 @@ -CREATE TABLE MyTable( - channel varchar as sss not null, - pv INT not null, - xctime bigint, - CHARACTER_LENGTH(channel) as timeLeng, - WATERMARK FOR xctime AS withOffset(xctime,1000) - )WITH( - type='kafka11', - bootstrapServers='172.16.8.107:9092', - offsetReset='latest', - topic='toutou' - ); - -CREATE TABLE MyResult( - channel varchar, - pv INT - )WITH( - type='mysql', - url='jdbc:mysql://172.16.8.109:3306/test?charset=utf8', - userName='dtstack', - password='abc123', - tableName='pv' - ); - - -insert -into - MyResult - select - a.channel, - a.pv - from - MyTable a - diff --git a/launcher/pom.xml b/launcher/pom.xml index 4c6334411..2f8140fd5 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -33,7 +33,7 @@ com.alibaba fastjson - 1.2.25 + 1.2.7 diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java index ffabc7002..9dfe2e07e 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/ClusterClientFactory.java @@ -18,68 +18,59 @@ package com.dtstack.flink.sql.launcher; -import com.dtstack.flink.sql.option.Options; -import com.dtstack.flink.sql.util.PluginUtil; -import com.dtstack.flink.sql.yarn.JobParameter; -import com.dtstack.flink.sql.yarn.YarnClusterConfiguration; -import org.apache.commons.io.Charsets; +import com.dtstack.flink.sql.ClusterMode; import org.apache.commons.lang.StringUtils; +import org.apache.flink.client.deployment.StandaloneClusterDescriptor; +import org.apache.flink.client.deployment.StandaloneClusterId; import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.MiniClusterClient; +import org.apache.flink.client.program.rest.RestClusterClient; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.runtime.akka.AkkaUtils; -import org.apache.flink.runtime.minicluster.MiniCluster; -import org.apache.flink.runtime.minicluster.MiniClusterConfiguration; import org.apache.flink.runtime.util.LeaderConnectionInfo; import org.apache.flink.yarn.AbstractYarnClusterDescriptor; import org.apache.flink.yarn.YarnClusterDescriptor; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import java.io.File; -import java.net.InetSocketAddress; -import java.net.URLDecoder; -import java.util.*; - -import com.dtstack.flink.sql.enums.ClusterMode; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.StringHelper; -import java.io.IOException; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import static java.util.Objects.requireNonNull; +import java.net.InetSocketAddress; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Set; /** - * @author sishu.yss + * The Factory of ClusterClient + * + * Company: www.dtstack.com + * @author huyifan.zju@163.com */ public class ClusterClientFactory { - public static ClusterClient createClusterClient(Options launcherOptions) throws Exception { + public static ClusterClient createClusterClient(LauncherOptions launcherOptions) throws Exception { String mode = launcherOptions.getMode(); if(mode.equals(ClusterMode.standalone.name())) { return createStandaloneClient(launcherOptions); } else if(mode.equals(ClusterMode.yarn.name())) { - return createYarnClient(launcherOptions,mode); + return createYarnClient(launcherOptions); } + throw new IllegalArgumentException("Unsupported cluster client type: "); } - public static ClusterClient createStandaloneClient(Options launcherOptions) throws Exception { + public static ClusterClient createStandaloneClient(LauncherOptions launcherOptions) throws Exception { String flinkConfDir = launcherOptions.getFlinkconf(); Configuration config = GlobalConfiguration.loadConfiguration(flinkConfDir); - MiniClusterConfiguration.Builder configBuilder = new MiniClusterConfiguration.Builder(); - configBuilder.setConfiguration(config); - MiniCluster miniCluster = new MiniCluster(configBuilder.build()); - MiniClusterClient clusterClient = new MiniClusterClient(config, miniCluster); + + StandaloneClusterDescriptor standaloneClusterDescriptor = new StandaloneClusterDescriptor(config); + RestClusterClient clusterClient = standaloneClusterDescriptor.retrieve(StandaloneClusterId.getInstance()); + LeaderConnectionInfo connectionInfo = clusterClient.getClusterConnectionInfo(); InetSocketAddress address = AkkaUtils.getInetSocketAddressFromAkkaURL(connectionInfo.getAddress()); config.setString(JobManagerOptions.ADDRESS, address.getAddress().getHostName()); @@ -88,192 +79,65 @@ public static ClusterClient createStandaloneClient(Options launcherOptions) thro return clusterClient; } - public static ClusterClient createYarnClient(Options launcherOptions, String mode) { + public static ClusterClient createYarnClient(LauncherOptions launcherOptions) { String flinkConfDir = launcherOptions.getFlinkconf(); - Configuration flinkConf = GlobalConfiguration.loadConfiguration(flinkConfDir); + Configuration config = GlobalConfiguration.loadConfiguration(flinkConfDir); String yarnConfDir = launcherOptions.getYarnconf(); - YarnConfiguration yarnConf; if(StringUtils.isNotBlank(yarnConfDir)) { - try { - flinkConf.setString(ConfigConstants.PATH_HADOOP_CONFIG, yarnConfDir); - FileSystem.initialize(flinkConf); - - File dir = new File(yarnConfDir); - if(dir.exists() && dir.isDirectory()) { - yarnConf = loadYarnConfiguration(yarnConfDir); - - YarnClient yarnClient = YarnClient.createYarnClient(); - haYarnConf(yarnConf); - yarnClient.init(yarnConf); - yarnClient.start(); - String confProp = launcherOptions.getConfProp(); - confProp = URLDecoder.decode(confProp, Charsets.UTF_8.toString()); - System.out.println("confProp="+confProp); - Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class); - - ApplicationId applicationId = null; - ClusterClient clusterClient = null; - if(mode.equals(ClusterMode.yarn.name())) {//on yarn cluster mode - String yarnSessionConf = launcherOptions.getYarnSessionConf(); - yarnSessionConf = URLDecoder.decode(yarnSessionConf, Charsets.UTF_8.toString()); - Properties yarnSessionConfProperties = PluginUtil.jsonStrToObject(yarnSessionConf, Properties.class); - String yid = yarnSessionConfProperties.get("yid").toString(); - if(StringUtils.isNotBlank(yid)){ - applicationId = toApplicationId(yid); - }else{ - applicationId = getYarnClusterApplicationId(yarnClient); - } - System.out.println("applicationId="+applicationId.toString()); - - AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor( - flinkConf, yarnConf, ".", yarnClient, false); - clusterClient = clusterDescriptor.retrieve(applicationId); + try { + config.setString(ConfigConstants.PATH_HADOOP_CONFIG, yarnConfDir); + FileSystem.initialize(config); + + YarnConfiguration yarnConf = YarnConfLoader.getYarnConf(yarnConfDir); + YarnClient yarnClient = YarnClient.createYarnClient(); + yarnClient.init(yarnConf); + yarnClient.start(); + ApplicationId applicationId = null; + + Set set = new HashSet<>(); + set.add("Apache Flink"); + EnumSet enumSet = EnumSet.noneOf(YarnApplicationState.class); + enumSet.add(YarnApplicationState.RUNNING); + List reportList = yarnClient.getApplications(set, enumSet); + + int maxMemory = -1; + int maxCores = -1; + for(ApplicationReport report : reportList) { + if(!report.getName().startsWith("Flink session")){ + continue; + } - System.out.println("applicationId="+applicationId.toString()+" has retrieve!"); - } else {//on yarn per-job mode - applicationId = createApplication(yarnClient); - System.out.println("applicationId="+applicationId.toString()); + if(!report.getYarnApplicationState().equals(YarnApplicationState.RUNNING)) { + continue; + } - YarnClusterConfiguration clusterConf = getYarnClusterConfiguration(flinkConf,yarnConf,flinkConfDir); - //jobmanager+taskmanager param - JobParameter appConf = new JobParameter(confProperties); + int thisMemory = report.getApplicationResourceUsageReport().getNeededResources().getMemory(); + int thisCores = report.getApplicationResourceUsageReport().getNeededResources().getVirtualCores(); + if(thisMemory > maxMemory || thisMemory == maxMemory && thisCores > maxCores) { + maxMemory = thisMemory; + maxCores = thisCores; + applicationId = report.getApplicationId(); + } - com.dtstack.flink.sql.yarn.YarnClusterDescriptor clusterDescriptor = new com.dtstack.flink.sql.yarn.YarnClusterDescriptor( - clusterConf, yarnClient, appConf,applicationId, launcherOptions.getName(),null ); - clusterClient = clusterDescriptor.deploy(); + } - System.out.println("applicationId="+applicationId.toString()+" has deploy!"); - } - clusterClient.setDetached(true); - yarnClient.stop(); - return clusterClient; + if(StringUtils.isEmpty(applicationId.toString())) { + throw new RuntimeException("No flink session found on yarn cluster."); } + + AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(config, yarnConf, ".", yarnClient, false); + ClusterClient clusterClient = clusterDescriptor.retrieve(applicationId); + clusterClient.setDetached(true); + return clusterClient; } catch(Exception e) { throw new RuntimeException(e); } } - throw new UnsupportedOperationException("Haven't been developed yet!"); - } - - private static YarnConfiguration loadYarnConfiguration(String yarnConfDir) - { - org.apache.hadoop.conf.Configuration hadoopConf = new org.apache.hadoop.conf.Configuration(); - hadoopConf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); - - Stream.of("yarn-site.xml", "core-site.xml", "hdfs-site.xml").forEach(file -> { - File site = new File(requireNonNull(yarnConfDir, "ENV HADOOP_CONF_DIR is not setting"), file); - if (site.exists() && site.isFile()) { - hadoopConf.addResource(new org.apache.hadoop.fs.Path(site.toURI())); - } - else { - throw new RuntimeException(site + " not exists"); - } - }); - - YarnConfiguration yarnConf = new YarnConfiguration(hadoopConf); - // try (PrintWriter pw = new PrintWriter(new FileWriter(yarnSite))) { //write local file - // yarnConf.writeXml(pw); - // } - return yarnConf; - } - - public static YarnClusterConfiguration getYarnClusterConfiguration(Configuration flinkConf,YarnConfiguration yarnConf,String flinkConfDir) - { - Path flinkJar = new Path(getFlinkJarFile(flinkConfDir).toURI()); - @SuppressWarnings("ConstantConditions") final Set resourcesToLocalize = Stream - .of("flink-conf.yaml", "log4j.properties") - .map(x -> new Path(new File(flinkConfDir, x).toURI())) - .collect(Collectors.toSet()); - - return new YarnClusterConfiguration(flinkConf, yarnConf, "", flinkJar, resourcesToLocalize); - } - - public static final String FLINK_DIST = "flink-dist"; - private static File getFlinkJarFile(String flinkConfDir) - { - String errorMessage = "error not search " + FLINK_DIST + "*.jar"; - File[] files = requireNonNull(new File(flinkConfDir, "/../lib").listFiles(), errorMessage); - Optional file = Arrays.stream(files) - .filter(f -> f.getName().startsWith(FLINK_DIST)).findFirst(); - return file.orElseThrow(() -> new IllegalArgumentException(errorMessage)); - } - - private static ApplicationId createApplication(YarnClient yarnClient)throws IOException, YarnException { - YarnClientApplication app = yarnClient.createApplication(); - return app.getApplicationSubmissionContext().getApplicationId(); - } - private static ApplicationId getYarnClusterApplicationId(YarnClient yarnClient) throws Exception{ - ApplicationId applicationId = null; - - Set set = new HashSet<>(); - set.add("Apache Flink"); - EnumSet enumSet = EnumSet.noneOf(YarnApplicationState.class); - enumSet.add(YarnApplicationState.RUNNING); - List reportList = yarnClient.getApplications(set, enumSet); - - int maxMemory = -1; - int maxCores = -1; - for(ApplicationReport report : reportList) { - if(!report.getName().startsWith("Flink session")){ - continue; - } - - if(!report.getYarnApplicationState().equals(YarnApplicationState.RUNNING)) { - continue; - } - - int thisMemory = report.getApplicationResourceUsageReport().getNeededResources().getMemory(); - int thisCores = report.getApplicationResourceUsageReport().getNeededResources().getVirtualCores(); - if(thisMemory > maxMemory || thisMemory == maxMemory && thisCores > maxCores) { - maxMemory = thisMemory; - maxCores = thisCores; - applicationId = report.getApplicationId(); - } - - } - if(StringUtils.isEmpty(applicationId.toString())) { - throw new RuntimeException("No flink session found on yarn cluster."); - } - return applicationId; - } - - /** - * 处理yarn HA的配置项 - */ - private static org.apache.hadoop.conf.Configuration haYarnConf(org.apache.hadoop.conf.Configuration yarnConf) { - Iterator> iterator = yarnConf.iterator(); - while(iterator.hasNext()) { - Map.Entry entry = iterator.next(); - String key = entry.getKey(); - String value = entry.getValue(); - if(key.startsWith("yarn.resourcemanager.hostname.")) { - String rm = key.substring("yarn.resourcemanager.hostname.".length()); - String addressKey = "yarn.resourcemanager.address." + rm; - if(yarnConf.get(addressKey) == null) { - yarnConf.set(addressKey, value + ":" + YarnConfiguration.DEFAULT_RM_PORT); - } - } - } - return yarnConf; + throw new UnsupportedOperationException("Haven't been developed yet!"); } - private static ApplicationId toApplicationId(String appIdStr) { - Iterator it = StringHelper._split(appIdStr).iterator(); - if (!(it.next()).equals("application")) { - throw new IllegalArgumentException("Invalid ApplicationId prefix: " + appIdStr + ". The valid ApplicationId should start with prefix " + "application"); - } else { - try { - return toApplicationId(it); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Invalid AppAttemptId: " + appIdStr, e); - } - } - } - private static ApplicationId toApplicationId(Iterator it) throws NumberFormatException { - return ApplicationId.newInstance(Long.parseLong(it.next()), Integer.parseInt(it.next())); - } -} \ No newline at end of file +} diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java index 0edd01434..f2d884778 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java @@ -20,17 +20,13 @@ package com.dtstack.flink.sql.launcher; -import com.google.common.collect.Lists; +import avro.shaded.com.google.common.collect.Lists; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.TypeReference; -import com.dtstack.flink.sql.enums.ClusterMode; +import com.dtstack.flink.sql.ClusterMode; import com.dtstack.flink.sql.Main; import com.dtstack.flink.sql.launcher.perjob.PerJobSubmitter; -import com.dtstack.flink.sql.option.OptionParser; -import com.dtstack.flink.sql.option.Options; import com.dtstack.flink.sql.util.PluginUtil; -import org.apache.commons.lang.BooleanUtils; -import org.apache.commons.lang.StringUtils; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.PackagedProgram; import org.apache.flink.client.program.PackagedProgramUtils; @@ -38,6 +34,8 @@ import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; +import org.apache.flink.table.shaded.org.apache.commons.lang.BooleanUtils; +import org.apache.flink.table.shaded.org.apache.commons.lang.StringUtils; import java.io.BufferedReader; import java.io.File; @@ -70,8 +68,9 @@ public static void main(String[] args) throws Exception { if (args.length == 1 && args[0].endsWith(".json")){ args = parseJson(args); } - OptionParser optionParser = new OptionParser(args); - Options launcherOptions = optionParser.getOptions(); + + LauncherOptionParser optionParser = new LauncherOptionParser(args); + LauncherOptions launcherOptions = optionParser.getLauncherOptions(); String mode = launcherOptions.getMode(); List argList = optionParser.getProgramExeArgList(); @@ -101,6 +100,7 @@ public static void main(String[] args) throws Exception { clusterClient.shutdown(); } + System.out.println("---submit end----"); } private static String[] parseJson(String[] args) { diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherOptionParser.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherOptionParser.java new file mode 100644 index 000000000..c658cd895 --- /dev/null +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherOptionParser.java @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dtstack.flink.sql.launcher; + +import avro.shaded.com.google.common.collect.Lists; +import org.apache.commons.cli.BasicParser; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.lang.StringUtils; +import org.apache.flink.hadoop.shaded.com.google.common.base.Charsets; +import org.apache.flink.hadoop.shaded.com.google.common.base.Preconditions; +import com.dtstack.flink.sql.util.PluginUtil; +import java.io.File; +import java.io.FileInputStream; +import java.net.URLEncoder; +import java.util.List; +import java.util.Map; +import com.dtstack.flink.sql.ClusterMode; + + + +/** + * The Parser of Launcher commandline options + * + * Company: www.dtstack.com + * @author sishu.yss + */ +public class LauncherOptionParser { + + public static final String OPTION_MODE = "mode"; + + public static final String OPTION_NAME = "name"; + + public static final String OPTION_SQL = "sql"; + + public static final String OPTION_FLINK_CONF_DIR = "flinkconf"; + + public static final String OPTION_YARN_CONF_DIR = "yarnconf"; + + public static final String OPTION_LOCAL_SQL_PLUGIN_PATH = "localSqlPluginPath"; + + public static final String OPTION_REMOTE_SQL_PLUGIN_PATH = "remoteSqlPluginPath"; + + public static final String OPTION_ADDJAR = "addjar"; + + public static final String OPTION_CONF_PROP = "confProp"; + + public static final String OPTION_SAVE_POINT_PATH = "savePointPath"; + + public static final String OPTION_ALLOW_NON_RESTORED_STATE = "allowNonRestoredState"; + + public static final String OPTION_FLINK_JAR_PATH = "flinkJarPath"; + + public static final String OPTION_QUEUE = "queue"; + + private Options options = new Options(); + + private BasicParser parser = new BasicParser(); + + private LauncherOptions properties = new LauncherOptions(); + + public LauncherOptionParser(String[] args) { + options.addOption(OPTION_MODE, true, "Running mode"); + options.addOption(OPTION_SQL, true, "Job sql file"); + options.addOption(OPTION_NAME, true, "Job name"); + options.addOption(OPTION_FLINK_CONF_DIR, true, "Flink configuration directory"); + options.addOption(OPTION_LOCAL_SQL_PLUGIN_PATH, true, "sql local plugin root"); + options.addOption(OPTION_REMOTE_SQL_PLUGIN_PATH, true, "sql remote plugin root"); + options.addOption(OPTION_ADDJAR, true, "sql ext jar,eg udf jar"); + options.addOption(OPTION_CONF_PROP, true, "sql ref prop,eg specify event time"); + options.addOption(OPTION_YARN_CONF_DIR, true, "Yarn and hadoop configuration directory"); + + options.addOption(OPTION_SAVE_POINT_PATH, true, "Savepoint restore path"); + options.addOption(OPTION_ALLOW_NON_RESTORED_STATE, true, "Flag indicating whether non restored state is allowed if the savepoint"); + options.addOption(OPTION_FLINK_JAR_PATH, true, "flink jar path for submit of perjob mode"); + options.addOption(OPTION_QUEUE, true, "flink runing yarn queue"); + + try { + CommandLine cl = parser.parse(options, args); + String mode = cl.getOptionValue(OPTION_MODE, ClusterMode.local.name()); + //check mode + properties.setMode(mode); + + String job = Preconditions.checkNotNull(cl.getOptionValue(OPTION_SQL), + "Must specify job file using option '" + OPTION_SQL + "'"); + File file = new File(job); + FileInputStream in = new FileInputStream(file); + byte[] filecontent = new byte[(int) file.length()]; + in.read(filecontent); + String content = new String(filecontent, "UTF-8"); + String sql = URLEncoder.encode(content, Charsets.UTF_8.name()); + properties.setSql(sql); + String localPlugin = Preconditions.checkNotNull(cl.getOptionValue(OPTION_LOCAL_SQL_PLUGIN_PATH)); + properties.setLocalSqlPluginPath(localPlugin); + String remotePlugin = cl.getOptionValue(OPTION_REMOTE_SQL_PLUGIN_PATH); +// if(!ClusterMode.local.name().equals(mode)){ +// Preconditions.checkNotNull(remotePlugin); + properties.setRemoteSqlPluginPath(remotePlugin); +// } + String name = Preconditions.checkNotNull(cl.getOptionValue(OPTION_NAME)); + properties.setName(name); + String addJar = cl.getOptionValue(OPTION_ADDJAR); + if(StringUtils.isNotBlank(addJar)){ + properties.setAddjar(addJar); + } + String confProp = cl.getOptionValue(OPTION_CONF_PROP); + if(StringUtils.isNotBlank(confProp)){ + properties.setConfProp(confProp); + } + String flinkConfDir = cl.getOptionValue(OPTION_FLINK_CONF_DIR); + if(StringUtils.isNotBlank(flinkConfDir)) { + properties.setFlinkconf(flinkConfDir); + } + + String yarnConfDir = cl.getOptionValue(OPTION_YARN_CONF_DIR); + if(StringUtils.isNotBlank(yarnConfDir)) { + properties.setYarnconf(yarnConfDir); + } + + String savePointPath = cl.getOptionValue(OPTION_SAVE_POINT_PATH); + if(StringUtils.isNotBlank(savePointPath)) { + properties.setSavePointPath(savePointPath); + } + + String allow_non = cl.getOptionValue(OPTION_ALLOW_NON_RESTORED_STATE); + if(StringUtils.isNotBlank(allow_non)) { + properties.setAllowNonRestoredState(allow_non); + } + + String flinkJarPath = cl.getOptionValue(OPTION_FLINK_JAR_PATH); + if(StringUtils.isNotBlank(flinkJarPath)){ + properties.setFlinkJarPath(flinkJarPath); + } + + String queue = cl.getOptionValue(OPTION_QUEUE); + if(StringUtils.isNotBlank(queue)){ + properties.setQueue(queue); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public LauncherOptions getLauncherOptions(){ + return properties; + } + + public List getProgramExeArgList() throws Exception { + Map mapConf = PluginUtil.ObjectToMap(properties); + List args = Lists.newArrayList(); + for(Map.Entry one : mapConf.entrySet()){ + String key = one.getKey(); + if(OPTION_FLINK_CONF_DIR.equalsIgnoreCase(key) + || OPTION_YARN_CONF_DIR.equalsIgnoreCase(key) + || OPTION_FLINK_JAR_PATH.equalsIgnoreCase(key) + || OPTION_QUEUE.equalsIgnoreCase(key)){ + continue; + } + + if(one.getValue() == null){ + continue; + } + + args.add("-" + key); + args.add(one.getValue().toString()); + } + return args; + } +} diff --git a/core/src/main/java/com/dtstack/flink/sql/option/Options.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherOptions.java similarity index 65% rename from core/src/main/java/com/dtstack/flink/sql/option/Options.java rename to launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherOptions.java index a653aa42e..5cc762ebd 100644 --- a/core/src/main/java/com/dtstack/flink/sql/option/Options.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherOptions.java @@ -16,11 +16,7 @@ * limitations under the License. */ -package com.dtstack.flink.sql.option; - -import com.dtstack.flink.sql.enums.ClusterMode; -import com.dtstack.flink.sql.enums.EPluginLoadMode; - +package com.dtstack.flink.sql.launcher; /** * This class define commandline options for the Launcher program @@ -28,52 +24,34 @@ * Company: www.dtstack.com * @author huyifan.zju@163.com */ -public class Options { +public class LauncherOptions { - @OptionRequired(description = "Running mode") - private String mode = ClusterMode.local.name(); + private String mode; - @OptionRequired(required = true,description = "Job name") private String name; - @OptionRequired(required = true,description = "Job sql file") private String sql; - @OptionRequired(description = "Flink configuration directory") private String flinkconf; - @OptionRequired(description = "Yarn and Hadoop configuration directory") private String yarnconf; - @OptionRequired(required = true,description = "Sql local plugin root") private String localSqlPluginPath; - @OptionRequired(required = true,description = "Sql remote plugin root") private String remoteSqlPluginPath ; - @OptionRequired(description = "sql ext jar,eg udf jar") private String addjar; - @OptionRequired(description = "sql ref prop,eg specify event time") - private String confProp = "{}"; + private String confProp; - @OptionRequired(description = "Savepoint restore path") private String savePointPath; - @OptionRequired(description = "Flag indicating whether non restored state is allowed if the savepoint") private String allowNonRestoredState = "false"; - @OptionRequired(description = "flink jar path for submit of perjob mode") + //just use for per_job mode private String flinkJarPath; - @OptionRequired(description = "yarn queue") - private String queue = "default"; - - @OptionRequired(description = "yarn session configuration,such as yid") - private String yarnSessionConf = "{}"; - - @OptionRequired(description = "plugin load mode, by classpath or shipfile") - private String pluginLoadMode = EPluginLoadMode.CLASSPATH.name(); + private String queue; public String getMode() { return mode; @@ -178,20 +156,4 @@ public String getQueue() { public void setQueue(String queue) { this.queue = queue; } - - public String getYarnSessionConf() { - return yarnSessionConf; - } - - public void setYarnSessionConf(String yarnSessionConf) { - this.yarnSessionConf = yarnSessionConf; - } - - public String getPluginLoadMode() { - return pluginLoadMode; - } - - public void setPluginLoadMode(String pluginLoadMode) { - this.pluginLoadMode = pluginLoadMode; - } } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java index 5dc0971bc..8ddcd541d 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobClusterClientBuilder.java @@ -18,14 +18,10 @@ package com.dtstack.flink.sql.launcher.perjob; -import com.dtstack.flink.sql.enums.EPluginLoadMode; import com.dtstack.flink.sql.launcher.YarnConfLoader; -import com.dtstack.flink.sql.option.Options; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.configuration.Configuration; -import com.google.common.base.Strings; -import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.hadoop.shaded.com.google.common.base.Strings; import org.apache.flink.yarn.AbstractYarnClusterDescriptor; import org.apache.flink.yarn.YarnClusterDescriptor; import org.apache.hadoop.fs.Path; @@ -37,7 +33,6 @@ import java.net.URL; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Properties; /** @@ -48,6 +43,7 @@ */ public class PerJobClusterClientBuilder { + private YarnClient yarnClient; private YarnConfiguration yarnConf; @@ -65,71 +61,44 @@ public void init(String yarnConfDir){ System.out.println("----init yarn success ----"); } - public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(Properties confProp, String flinkJarPath, Options launcherOptions, JobGraph jobGraph) throws MalformedURLException { + public AbstractYarnClusterDescriptor createPerJobClusterDescriptor(Properties confProp, String flinkJarPath, String queue) throws MalformedURLException { Configuration newConf = new Configuration(); - confProp.forEach((key, val) -> newConf.setString(key.toString(), val.toString())); + confProp.forEach((key, val) -> newConf.setString(key.toString(), val.toString()) ); AbstractYarnClusterDescriptor clusterDescriptor = getClusterDescriptor(newConf, yarnConf, "."); if (StringUtils.isNotBlank(flinkJarPath)) { + if (!new File(flinkJarPath).exists()) { throw new RuntimeException("The Flink jar path is not exist"); } + } - List shipFiles = new ArrayList<>(); + List classpaths = new ArrayList<>(); if (flinkJarPath != null) { File[] jars = new File(flinkJarPath).listFiles(); - for (File file : jars) { - if (file.toURI().toURL().toString().contains("flink-dist")) { + + for (File file : jars){ + if (file.toURI().toURL().toString().contains("flink-dist")){ clusterDescriptor.setLocalJarPath(new Path(file.toURI().toURL().toString())); } else { - shipFiles.add(file); + classpaths.add(file.toURI().toURL()); } } + } else { throw new RuntimeException("The Flink jar path is null"); } - // classpath , all node need contain plugin jar - String pluginLoadMode = launcherOptions.getPluginLoadMode(); - if (StringUtils.equalsIgnoreCase(pluginLoadMode, EPluginLoadMode.CLASSPATH.name())) { - fillJobGraphClassPath(jobGraph); - } else if (StringUtils.equalsIgnoreCase(pluginLoadMode, EPluginLoadMode.SHIPFILE.name())) { - List pluginPaths = getPluginPathToShipFiles(jobGraph); - shipFiles.addAll(pluginPaths); - } else { - throw new IllegalArgumentException("Unsupported plugin loading mode " + pluginLoadMode - + " Currently only classpath and shipfile are supported."); - } - clusterDescriptor.addShipFiles(shipFiles); - String queue = launcherOptions.getQueue(); - if (!Strings.isNullOrEmpty(queue)) { + clusterDescriptor.setProvidedUserJarFiles(classpaths); + + if(!Strings.isNullOrEmpty(queue)){ clusterDescriptor.setQueue(queue); } return clusterDescriptor; } - private static void fillJobGraphClassPath(JobGraph jobGraph) throws MalformedURLException { - Map jobCacheFileConfig = jobGraph.getUserArtifacts(); - for(Map.Entry tmp : jobCacheFileConfig.entrySet()){ - if(tmp.getKey().startsWith("class_path")){ - jobGraph.getClasspaths().add(new URL("file:" + tmp.getValue().filePath)); - } - } - } - - private List getPluginPathToShipFiles(JobGraph jobGraph) { - List shipFiles = new ArrayList<>(); - Map jobCacheFileConfig = jobGraph.getUserArtifacts(); - for(Map.Entry tmp : jobCacheFileConfig.entrySet()){ - if(tmp.getKey().startsWith("class_path")){ - shipFiles.add(new File(tmp.getValue().filePath)); - } - } - return shipFiles; - } - private AbstractYarnClusterDescriptor getClusterDescriptor( Configuration configuration, YarnConfiguration yarnConfiguration, diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java index 55b55be2f..0708b4957 100644 --- a/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java +++ b/launcher/src/main/java/com/dtstack/flink/sql/launcher/perjob/PerJobSubmitter.java @@ -18,7 +18,7 @@ package com.dtstack.flink.sql.launcher.perjob; -import com.dtstack.flink.sql.option.Options; +import com.dtstack.flink.sql.launcher.LauncherOptions; import com.dtstack.flink.sql.util.PluginUtil; import org.apache.commons.io.Charsets; import org.apache.commons.lang3.StringUtils; @@ -26,14 +26,17 @@ import org.apache.flink.client.program.ClusterClient; import org.apache.flink.core.fs.Path; import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.shaded.guava18.com.google.common.base.Strings; +import org.apache.flink.shaded.guava18.com.google.common.collect.Sets; import org.apache.flink.yarn.AbstractYarnClusterDescriptor; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import java.net.MalformedURLException; +import java.net.URL; import java.net.URLDecoder; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; +import java.util.*; /** * per job mode submitter @@ -46,13 +49,17 @@ public class PerJobSubmitter { private static final Logger LOG = LoggerFactory.getLogger(PerJobSubmitter.class); - public static String submit(Options launcherOptions, JobGraph jobGraph) throws Exception { - if (!StringUtils.isBlank(launcherOptions.getAddjar())) { - String addjarPath = URLDecoder.decode(launcherOptions.getAddjar(), Charsets.UTF_8.toString()); - List paths = getJarPaths(addjarPath); - paths.forEach( path -> { + public static String submit(LauncherOptions launcherOptions, JobGraph jobGraph) throws Exception { + + fillJobGraphClassPath(jobGraph); + + String addjarPath = URLDecoder.decode(launcherOptions.getAddjar(), Charsets.UTF_8.toString()); + if (StringUtils.isNotBlank(addjarPath) ){ + List paths = getJarPaths(addjarPath); + paths.forEach( path ->{ jobGraph.addJar(new Path("file://" + path)); }); + } String confProp = launcherOptions.getConfProp(); @@ -65,7 +72,7 @@ public static String submit(Options launcherOptions, JobGraph jobGraph) throws E String flinkJarPath = launcherOptions.getFlinkJarPath(); - AbstractYarnClusterDescriptor yarnClusterDescriptor = perJobClusterClientBuilder.createPerJobClusterDescriptor(confProperties, flinkJarPath, launcherOptions, jobGraph); + AbstractYarnClusterDescriptor yarnClusterDescriptor = perJobClusterClientBuilder.createPerJobClusterDescriptor(confProperties, flinkJarPath, launcherOptions.getQueue()); ClusterClient clusterClient = yarnClusterDescriptor.deployJobCluster(clusterSpecification, jobGraph,true); String applicationId = clusterClient.getClusterId().toString(); @@ -86,4 +93,32 @@ private static List getJarPaths(String addjarPath) { return paths; } + private static void fillJobGraphClassPath(JobGraph jobGraph) throws MalformedURLException { + Map jobCacheFileConfig = jobGraph.getJobConfiguration().toMap(); + Set classPathKeySet = Sets.newHashSet(); + + for(Map.Entry tmp : jobCacheFileConfig.entrySet()){ + if(Strings.isNullOrEmpty(tmp.getValue())){ + continue; + } + + if(tmp.getValue().startsWith("class_path")){ + //DISTRIBUTED_CACHE_FILE_NAME_1 + //DISTRIBUTED_CACHE_FILE_PATH_1 + String key = tmp.getKey(); + String[] array = key.split("_"); + if(array.length < 5){ + continue; + } + + array[3] = "PATH"; + classPathKeySet.add(StringUtils.join(array, "_")); + } + } + + for(String key : classPathKeySet){ + String pathStr = jobCacheFileConfig.get(key); + jobGraph.getClasspaths().add(new URL("file:" + pathStr)); + } + } } diff --git a/launcher/src/main/java/com/dtstack/flink/sql/yarn/JobParameter.java b/launcher/src/main/java/com/dtstack/flink/sql/yarn/JobParameter.java deleted file mode 100644 index abfa0a8ad..000000000 --- a/launcher/src/main/java/com/dtstack/flink/sql/yarn/JobParameter.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (C) 2018 The Sylph Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.flink.sql.yarn; - -import java.util.Objects; -import java.util.Properties; - -public class JobParameter -{ - private int parallelism = 1; - private String queue = "default"; - private int taskManagerMemoryMb = 1024; - private int taskManagerCount = 1; - private int taskManagerSlots = 1; - private int jobManagerMemoryMb = 1024; - - public JobParameter() {} - - public JobParameter(Properties confProperties) { - this.parallelism = confProperties.getProperty("parallelism")==null?parallelism:Integer.parseInt(confProperties.getProperty("parallelism")); - this.queue = confProperties.getProperty("queue")==null?queue:confProperties.getProperty("queue"); - this.taskManagerMemoryMb = confProperties.getProperty("taskManagerMemoryMb")==null?taskManagerMemoryMb:Integer.parseInt(confProperties.getProperty("taskManagerMemoryMb")); - this.taskManagerCount = confProperties.getProperty("taskManagerCount")==null?taskManagerCount:Integer.parseInt(confProperties.getProperty("taskManagerCount")); - this.taskManagerSlots = confProperties.getProperty("taskManagerSlots")==null?taskManagerSlots:Integer.parseInt(confProperties.getProperty("taskManagerSlots")); - this.jobManagerMemoryMb = confProperties.getProperty("jobManagerMemoryMb")==null?jobManagerMemoryMb:Integer.parseInt(confProperties.getProperty("jobManagerMemoryMb")); - } - - public JobParameter(int parallelism, String queue, int taskManagerMemoryMb, int taskManagerCount, int taskManagerSlots, int jobManagerMemoryMb) { - this.parallelism = parallelism; - this.queue = queue; - this.taskManagerMemoryMb = taskManagerMemoryMb; - this.taskManagerCount = taskManagerCount; - this.taskManagerSlots = taskManagerSlots; - this.jobManagerMemoryMb = jobManagerMemoryMb; - } - - public void setQueue(String queue) - { - this.queue = queue; - } - - public void setTaskManagerCount(int taskManagerCount) - { - this.taskManagerCount = taskManagerCount; - } - - public void setTaskManagerMemoryMb(int taskManagerMemoryMb) - { - this.taskManagerMemoryMb = taskManagerMemoryMb; - } - - public void setTaskManagerSlots(int taskManagerSlots) - { - this.taskManagerSlots = taskManagerSlots; - } - - public void setJobManagerMemoryMb(int jobManagerMemoryMb) - { - this.jobManagerMemoryMb = jobManagerMemoryMb; - } - - public void setParallelism(int parallelism) - { - this.parallelism = parallelism; - } - - public int getParallelism() - { - return parallelism; - } - - public String getQueue() - { - return queue; - } - - public int getJobManagerMemoryMb() - { - return jobManagerMemoryMb; - } - - public int getTaskManagerSlots() - { - return taskManagerSlots; - } - - public int getTaskManagerCount() - { - return taskManagerCount; - } - - public int getTaskManagerMemoryMb() - { - return taskManagerMemoryMb; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - JobParameter jobParameter = (JobParameter) o; - return Objects.equals(this.queue, jobParameter.queue) && - Objects.equals(this.taskManagerCount, jobParameter.taskManagerCount) && - Objects.equals(this.taskManagerMemoryMb, jobParameter.taskManagerMemoryMb); - } - - @Override - public int hashCode() - { - return Objects.hash(queue, taskManagerMemoryMb, taskManagerCount); - } -} diff --git a/launcher/src/main/java/com/dtstack/flink/sql/yarn/YarnClusterConfiguration.java b/launcher/src/main/java/com/dtstack/flink/sql/yarn/YarnClusterConfiguration.java deleted file mode 100644 index 2189e9a9a..000000000 --- a/launcher/src/main/java/com/dtstack/flink/sql/yarn/YarnClusterConfiguration.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2018 The Sylph Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.flink.sql.yarn; - -import org.apache.flink.configuration.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.yarn.conf.YarnConfiguration; - -import java.util.Set; - -public class YarnClusterConfiguration { - /** - * The configuration used by YARN (i.e., 
yarn-site.xml
). - */ - private final YarnConfiguration yarnConf; - - /** - * The home directory of all job where all the temporary files for each jobs are stored. - */ - private final String appRootDir; - - /** - * Thepublic YarnClusterConfiguration(Configuration , YarnConfiguration , String , Path , Set ) { - } location of the Flink jar. - */ - private final Path flinkJar; - - /** - * Additional resources to be localized for both JobManager and TaskManager. - * They will NOT be added into the classpaths. - */ - private final Set resourcesToLocalize; - - /** - * flink conf - */ - private final Configuration flinkConfiguration; - - public YarnClusterConfiguration( - Configuration flinkConf, - YarnConfiguration conf, - String appRootDir, - Path flinkJar, - Set resourcesToLocalize) { - this.flinkConfiguration = flinkConf; - this.yarnConf = conf; - this.appRootDir = appRootDir; - this.flinkJar = flinkJar; - this.resourcesToLocalize = resourcesToLocalize; - } - - - - YarnConfiguration yarnConf() { - return yarnConf; - } - - public String appRootDir() { - return appRootDir; - } - - public Configuration flinkConfiguration() { - return flinkConfiguration; - } - - public Path flinkJar() { - return flinkJar; - } - - public Set resourcesToLocalize() { - return resourcesToLocalize; - } - -} diff --git a/launcher/src/main/java/com/dtstack/flink/sql/yarn/YarnClusterDescriptor.java b/launcher/src/main/java/com/dtstack/flink/sql/yarn/YarnClusterDescriptor.java deleted file mode 100644 index 989a4400a..000000000 --- a/launcher/src/main/java/com/dtstack/flink/sql/yarn/YarnClusterDescriptor.java +++ /dev/null @@ -1,396 +0,0 @@ -/* - * Copyright (C) 2018 The Sylph Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.flink.sql.yarn; - -import org.apache.flink.client.deployment.ClusterDeploymentException; -import org.apache.flink.client.deployment.ClusterSpecification; -import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.rest.RestClusterClient; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.JobManagerOptions; -import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.yarn.*; -import org.apache.flink.yarn.entrypoint.YarnSessionClusterEntrypoint; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.*; -import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.hadoop.yarn.util.Records; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.*; -import java.util.concurrent.*; -import java.util.stream.Collectors; - -import static java.util.Objects.requireNonNull; -import static org.apache.hadoop.yarn.api.records.YarnApplicationState.NEW; - -public class YarnClusterDescriptor - extends AbstractYarnClusterDescriptor -{ - private static final String APPLICATION_TYPE = "58_FLINK"; - private static final Logger LOG = LoggerFactory.getLogger(YarnClusterDescriptor.class); - private static final int MAX_ATTEMPT = 1; - private static final long DEPLOY_TIMEOUT_MS = 600 * 1000; - private static final long RETRY_DELAY_MS = 250; - private static final ScheduledExecutorService YARN_POLL_EXECUTOR = Executors.newSingleThreadScheduledExecutor(); - - private final YarnClusterConfiguration clusterConf; - private final YarnClient yarnClient; - private final JobParameter appConf; - private final Path homedir; - private final ApplicationId yarnAppId; - private final String jobName; - private final Iterable userProvidedJars; - private Path flinkJar; - - public YarnClusterDescriptor( - final YarnClusterConfiguration clusterConf, - final YarnClient yarnClient, - final JobParameter appConf, - ApplicationId yarnAppId, - String jobName, - Iterable userProvidedJars) - { - super(clusterConf.flinkConfiguration(), clusterConf.yarnConf(), clusterConf.appRootDir(), yarnClient, false); - this.jobName = jobName; - this.clusterConf = clusterConf; - this.yarnClient = yarnClient; - this.appConf = appConf; - this.yarnAppId = yarnAppId; - this.userProvidedJars = userProvidedJars; - this.homedir = new Path(clusterConf.appRootDir(), yarnAppId.toString()); - } - - @Override - protected String getYarnSessionClusterEntrypoint() - { - return YarnSessionClusterEntrypoint.class.getName(); - } - - /** - * 提交到yarn时 任务启动入口类 - */ - @Override - protected String getYarnJobClusterEntrypoint() - { - return YarnSessionClusterEntrypoint.class.getName(); - } - - @Override - protected ClusterClient createYarnClusterClient(AbstractYarnClusterDescriptor descriptor, int numberTaskManagers, int slotsPerTaskManager, ApplicationReport report, Configuration flinkConfiguration, boolean perJobCluster) - throws Exception - { - return new RestClusterClient<>( - flinkConfiguration, - report.getApplicationId()); - } - - @Override - public YarnClient getYarnClient() - { - return this.yarnClient; - } - - public RestClusterClient deploy() - { - ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class); - context.setApplicationId(yarnAppId); - try { - ApplicationReport report = startAppMaster(context); - - Configuration conf = getFlinkConfiguration(); - conf.setString(JobManagerOptions.ADDRESS.key(), report.getHost()); - conf.setInteger(JobManagerOptions.PORT.key(), report.getRpcPort()); - - /*return new RestClusterClient(this, - appConf.getTaskManagerCount(), - appConf.getTaskManagerSlots(), - report, conf, false);*/ - return new RestClusterClient<>( - conf, - report.getApplicationId()); - } - catch (Exception e) { - throw new RuntimeException(e); - } - } - - private ApplicationReport startAppMaster(ApplicationSubmissionContext appContext) - throws Exception - { - ApplicationId appId = appContext.getApplicationId(); - appContext.setMaxAppAttempts(MAX_ATTEMPT); - - Map localResources = new HashMap<>(); - Set shippedPaths = new HashSet<>(); - collectLocalResources(localResources, shippedPaths); - - final ContainerLaunchContext amContainer = setupApplicationMasterContainer( - getYarnJobClusterEntrypoint(), - false, - true, - false, - appConf.getJobManagerMemoryMb() - ); - - amContainer.setLocalResources(localResources); - - final String classPath = String.join(File.pathSeparator, localResources.keySet()); - - final String shippedFiles = shippedPaths.stream() - .map(path -> path.getName() + "=" + path) - .collect(Collectors.joining(",")); - - // Setup CLASSPATH and environment variables for ApplicationMaster - final Map appMasterEnv = setUpAmEnvironment( - appId, - classPath,shippedFiles, - //"","", - getDynamicPropertiesEncoded() - ); - - amContainer.setEnvironment(appMasterEnv); - - // Set up resource type requirements for ApplicationMaster - Resource capability = Records.newRecord(Resource.class); - capability.setMemory(appConf.getJobManagerMemoryMb()); //设置jobManneger - capability.setVirtualCores(1); //默认是1 - - appContext.setApplicationName(jobName); - appContext.setApplicationType(APPLICATION_TYPE); - appContext.setAMContainerSpec(amContainer); - appContext.setResource(capability); - //appContext.setApplicationTags(appConf.getAppTags()); - if (appConf.getQueue() != null) { - appContext.setQueue(appConf.getQueue()); - } - - LOG.info("Submitting application master {}", appId); - yarnClient.submitApplication(appContext); - - PollDeploymentStatus poll = new PollDeploymentStatus(appId); - YARN_POLL_EXECUTOR.submit(poll); - try { - return poll.result.get(); - } - catch (ExecutionException e) { - LOG.warn("Failed to deploy {}, cause: {}", appId.toString(), e.getCause()); - yarnClient.killApplication(appId); - throw (Exception) e.getCause(); - } - } - - private void collectLocalResources( - Map resources, - Set shippedPaths - ) - throws IOException, URISyntaxException - { - if(clusterConf.flinkJar() != null) { - Path flinkJar = clusterConf.flinkJar(); - LocalResource flinkJarResource = setupLocalResource(flinkJar, homedir, ""); //放到 Appid/根目录下 - this.flinkJar = ConverterUtils.getPathFromYarnURL(flinkJarResource.getResource()); - resources.put("flink.jar", flinkJarResource); - } - if(clusterConf.resourcesToLocalize() != null) { - for (Path p : clusterConf.resourcesToLocalize()) { //主要是 flink.jar log4f.propors 和 flink.yaml 三个文件 - LocalResource resource = setupLocalResource(p, homedir, ""); //这些需要放到根目录下 - resources.put(p.getName(), resource); - if ("log4j.properties".equals(p.getName())) { - shippedPaths.add(ConverterUtils.getPathFromYarnURL(resource.getResource())); - } - } - } - if(userProvidedJars != null) { - for (Path p : userProvidedJars) { - String name = p.getName(); - if (resources.containsKey(name)) { //这里当jar 有重复的时候 会抛出异常 - LOG.warn("Duplicated name in the shipped files {}", p); - } else { - LocalResource resource = setupLocalResource(p, homedir, "jars"); //这些放到 jars目录下 - resources.put(name, resource); - shippedPaths.add(ConverterUtils.getPathFromYarnURL(resource.getResource())); - } - } - } - } - - private LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) - throws IOException - { - LocalResource localResource = Records.newRecord(LocalResource.class); - FileStatus jarStat = fs.getFileStatus(remoteRsrcPath); - localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); - localResource.setSize(jarStat.getLen()); - localResource.setTimestamp(jarStat.getModificationTime()); - localResource.setType(LocalResourceType.FILE); - localResource.setVisibility(LocalResourceVisibility.APPLICATION); - return localResource; - } - - private LocalResource setupLocalResource( - Path localSrcPath, - Path homedir, - String relativeTargetPath) - throws IOException - { - if (new File(localSrcPath.toUri().getPath()).isDirectory()) { - throw new IllegalArgumentException("File to copy must not be a directory: " + - localSrcPath); - } - - // copy resource to HDFS - String suffix = "." + (relativeTargetPath.isEmpty() ? "" : "/" + relativeTargetPath) - + "/" + localSrcPath.getName(); - - Path dst = new Path(homedir, suffix); - - LOG.info("Uploading {}", dst); - - FileSystem hdfs = FileSystem.get(clusterConf.yarnConf()); - hdfs.copyFromLocalFile(false, true, localSrcPath, dst); - - // now create the resource instance - LocalResource resource = registerLocalResource(hdfs, dst); - return resource; - } - - private Map setUpAmEnvironment( - ApplicationId appId, - String amClassPath, - String shipFiles, - String dynamicProperties) - throws IOException, URISyntaxException - { - final Map appMasterEnv = new HashMap<>(); - - // set Flink app class path - appMasterEnv.put(YarnConfigKeys.ENV_FLINK_CLASSPATH, amClassPath); - - // set Flink on YARN internal configuration values - appMasterEnv.put(YarnConfigKeys.ENV_TM_COUNT, String.valueOf(appConf.getTaskManagerCount())); - appMasterEnv.put(YarnConfigKeys.ENV_TM_MEMORY, String.valueOf(appConf.getTaskManagerMemoryMb())); - appMasterEnv.put(YarnConfigKeys.ENV_SLOTS, String.valueOf(appConf.getTaskManagerSlots())); - appMasterEnv.put(YarnConfigKeys.FLINK_JAR_PATH, flinkJar.toString()); - appMasterEnv.put(YarnConfigKeys.ENV_APP_ID, appId.toString()); - appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, homedir.toString()); //$home/.flink/appid 这个目录里面存放临时数据 - appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, shipFiles); - - appMasterEnv.put(YarnConfigKeys.ENV_DETACHED, String.valueOf(true)); //是否分离 分离就cluser模式 否则是client模式 - - appMasterEnv.put(YarnConfigKeys.ENV_HADOOP_USER_NAME, - UserGroupInformation.getCurrentUser().getUserName()); - - if (dynamicProperties != null) { - appMasterEnv.put(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES, dynamicProperties); - } - - // set classpath from YARN configuration - Utils.setupYarnClassPath(clusterConf.yarnConf(), appMasterEnv); - - return appMasterEnv; - } - - /** - * flink 1.5 add - */ - @Override - public ClusterClient deployJobCluster(ClusterSpecification clusterSpecification, JobGraph jobGraph, boolean detached) - throws ClusterDeploymentException - { - throw new UnsupportedOperationException("this method have't support!"); - } - - private final class PollDeploymentStatus - implements Runnable - { - private final CompletableFuture result = new CompletableFuture<>(); - private final ApplicationId appId; - private YarnApplicationState lastAppState = NEW; - private long startTime; - - private PollDeploymentStatus(ApplicationId appId) - { - this.appId = appId; - } - - @Override - public void run() - { - if (startTime == 0) { - startTime = System.currentTimeMillis(); - } - - try { - ApplicationReport report = poll(); - if (report == null) { - YARN_POLL_EXECUTOR.schedule(this, RETRY_DELAY_MS, TimeUnit.MILLISECONDS); - } - else { - result.complete(report); - } - } - catch (YarnException | IOException e) { - result.completeExceptionally(e); - } - } - - private ApplicationReport poll() - throws IOException, YarnException - { - ApplicationReport report; - report = yarnClient.getApplicationReport(appId); - YarnApplicationState appState = report.getYarnApplicationState(); - LOG.debug("Application State: {}", appState); - - switch (appState) { - case FAILED: - case FINISHED: - //TODO: the finished state may be valid in flip-6 - case KILLED: - throw new IOException("The YARN application unexpectedly switched to state " - + appState + " during deployment. \n" - + "Diagnostics from YARN: " + report.getDiagnostics() + "\n" - + "If log aggregation is enabled on your cluster, use this command to further investigate the issue:\n" - + "yarn logs -applicationId " + appId); - //break .. - case RUNNING: - LOG.info("YARN application has been deployed successfully."); - break; - default: - if (appState != lastAppState) { - LOG.info("Deploying cluster, current state " + appState); - } - lastAppState = appState; - if (System.currentTimeMillis() - startTime > DEPLOY_TIMEOUT_MS) { - throw new RuntimeException(String.format("Deployment took more than %d seconds. " - + "Please check if the requested resources are available in the YARN cluster", DEPLOY_TIMEOUT_MS)); - } - return null; - } - return report; - } - } -} diff --git a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java b/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java deleted file mode 100644 index 0a153d018..000000000 --- a/launcher/src/main/test/java/com/dtstack/flink/sql/launcher/PluginLoadModeTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.launcher; - - -/** - * yarnPer提交任务时指定pluginLoadMode - * Date: 2019/11/6 - * Company: www.dtstack.com - * @author maqi - */ -public class PluginLoadModeTest { - public static void testShipfileMode() throws Exception { - String[] sql = new String[]{"--mode", "yarnPer", "-sql", "/Users/maqi/tmp/json/group_tmp4.txt", "-name", "PluginLoadModeTest", - "-localSqlPluginPath", "/Users/maqi/code/dtstack/dt-center-flinkStreamSQL/plugins", - "-remoteSqlPluginPath", "/Users/maqi/code/dtstack/dt-center-flinkStreamSQL/plugins", - "-allowNonRestoredState", "false", "-flinkconf", "/Users/maqi/tmp/flink-1.8.1/conf", - "-confProp", "{\"sql.checkpoint.cleanup.mode\":\"false\",\"sql.checkpoint.interval\":10000,\"time.characteristic\":\"EventTime\"}", - "-yarnconf", "/Users/maqi/tmp/hadoop", "-flinkJarPath", "/Users/maqi/tmp/flink-1.8.1/lib", "-queue", "c", "-pluginLoadMode", "shipfile"}; - System.setProperty("HADOOP_USER_NAME", "admin"); - LauncherMain.main(sql); - } - - public static void testClasspathMode() throws Exception { - String[] sql = new String[]{"--mode", "yarnPer", "-sql", "/Users/maqi/tmp/json/group_tmp4.txt", "-name", "PluginLoadModeTest", - "-localSqlPluginPath", "/Users/maqi/code/dtstack/dt-center-flinkStreamSQL/plugins", - "-remoteSqlPluginPath", "/opt/dtstack/180_flinkplugin/sqlplugin", - "-allowNonRestoredState", "false", "-flinkconf", "/Users/maqi/tmp/flink-1.8.1/conf", - "-confProp", "{\"sql.checkpoint.cleanup.mode\":\"false\",\"sql.checkpoint.interval\":10000,\"time.characteristic\":\"EventTime\"}", - "-yarnconf", "/Users/maqi/tmp/hadoop", "-flinkJarPath", "/Users/maqi/tmp/flink-1.8.1/lib", "-queue", "c", "-pluginLoadMode", "classpath"}; - System.setProperty("HADOOP_USER_NAME", "admin"); - LauncherMain.main(sql); - } - - public static void main(String[] args) throws Exception { - testShipfileMode(); -// testClasspathMode(); - } -} diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java index 4dc7c26b4..d1a38ca5f 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllReqRow.java @@ -36,8 +36,8 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Maps; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; diff --git a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java index 6191114b5..5d94531d2 100644 --- a/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java +++ b/mongo/mongo-side/mongo-all-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAllSideInfo.java @@ -23,11 +23,12 @@ import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; -import com.dtstack.flink.sql.util.ParseUtils; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import java.util.List; @@ -84,7 +85,11 @@ public void parseSelectFields(JoinInfo joinInfo){ SqlNode conditionNode = joinInfo.getCondition(); List sqlNodeList = Lists.newArrayList(); - ParseUtils.parseAnd(conditionNode, sqlNodeList); + if(conditionNode.getKind() == SqlKind.AND){ + sqlNodeList.addAll(Lists.newArrayList(((SqlBasicCall)conditionNode).getOperands())); + }else{ + sqlNodeList.add(conditionNode); + } for(SqlNode sqlNode : sqlNodeList){ dealOneEqualCon(sqlNode, sideTableName); diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java index 150053246..226885ed7 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncReqRow.java @@ -40,7 +40,7 @@ import com.mongodb.connection.ConnectionPoolSettings; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; diff --git a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java index 365026972..b09a12474 100644 --- a/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java +++ b/mongo/mongo-side/mongo-async-side/src/main/java/com/dtstack/flink/sql/side/mongo/MongoAsyncSideInfo.java @@ -23,13 +23,12 @@ import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.mongo.table.MongoSideTableInfo; -import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import java.util.List; @@ -56,7 +55,11 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { SqlNode conditionNode = joinInfo.getCondition(); List sqlNodeList = Lists.newArrayList(); - ParseUtils.parseAnd(conditionNode, sqlNodeList); + if(conditionNode.getKind() == SqlKind.AND){ + sqlNodeList.addAll(Lists.newArrayList(((SqlBasicCall)conditionNode).getOperands())); + }else{ + sqlNodeList.add(conditionNode); + } for(SqlNode sqlNode : sqlNodeList){ dealOneEqualCon(sqlNode, sideTableName); diff --git a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java index a5c834469..721960003 100644 --- a/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java +++ b/mongo/mongo-side/mongo-side-core/src/main/java/com/dtstack/flink/sql/side/mongo/table/MongoSideTableInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.side.mongo.table; import com.dtstack.flink.sql.side.SideTableInfo; -import com.google.common.base.Preconditions; +import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; /** * Reason: diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java index f37f5f379..87ffc7354 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/MongoOutputFormat.java @@ -63,8 +63,6 @@ public class MongoOutputFormat extends MetricOutputFormat { private static String PK = "_ID"; - private static int rowLenth = 1000; - public final SimpleDateFormat ROWKEY_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss"); @Override @@ -109,10 +107,6 @@ public void writeRecord(Tuple2 tuple2) throws IOException { } else { dbCollection.insertOne(doc); } - - if (outRecords.getCount()%rowLenth == 0){ - LOG.info(record.toString()); - } outRecords.inc(); } diff --git a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java index 02a96d6bb..91de9eba6 100644 --- a/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java +++ b/mongo/mongo-sink/src/main/java/com/dtstack/flink/sql/sink/mongo/table/MongoTableInfo.java @@ -20,7 +20,7 @@ package com.dtstack.flink.sql.sink.mongo.table; import com.dtstack.flink.sql.table.TargetTableInfo; -import com.google.common.base.Preconditions; +import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; /** * Reason: diff --git a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java index b6b7f45cb..aca7593f7 100644 --- a/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java +++ b/mysql/mysql-side/mysql-all-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAllReqRow.java @@ -24,7 +24,7 @@ import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Maps; +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +50,7 @@ public class MysqlAllReqRow extends RdbAllReqRow { private static final String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; public MysqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(new MysqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override diff --git a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java index 6120767ef..456451214 100644 --- a/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java +++ b/mysql/mysql-side/mysql-async-side/src/main/java/com/dtstack/flink/sql/side/mysql/MysqlAsyncReqRow.java @@ -50,7 +50,7 @@ public class MysqlAsyncReqRow extends RdbAsyncReqRow { private final static String MYSQL_DRIVER = "com.mysql.jdbc.Driver"; public MysqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(new MysqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @@ -63,18 +63,11 @@ public void open(Configuration parameters) throws Exception { .put("driver_class", MYSQL_DRIVER) .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) - .put("password", rdbSideTableInfo.getPassword()) - .put("provider_class", DT_PROVIDER_CLASS) - .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) - .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) - .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); - - System.setProperty("vertx.disableFileCPResolving", "true"); + .put("password", rdbSideTableInfo.getPassword()); VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); - vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, mysqlClientConfig)); } diff --git a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java index 9ba6736be..d045729e8 100644 --- a/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java +++ b/mysql/mysql-sink/src/main/java/com/dtstack/flink/sql/sink/mysql/MysqlSink.java @@ -23,8 +23,6 @@ import com.dtstack.flink.sql.sink.IStreamSinkGener; import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; -import com.dtstack.flink.sql.util.DtStringUtil; - import java.util.List; import java.util.Map; @@ -48,17 +46,17 @@ public RetractJDBCOutputFormat getOutputFormat() { } @Override - public void buildSql(String scheam, String tableName, List fields) { + public void buildSql(String tableName, List fields) { buildInsertSql(tableName, fields); } @Override - public String buildUpdateSql(String schema, String tableName, List fieldNames, Map> realIndexes, List fullField) { + public String buildUpdateSql(String tableName, List fieldNames, Map> realIndexes, List fullField) { return null; } private void buildInsertSql(String tableName, List fields) { - String sqlTmp = "replace into " + tableName + " (${fields}) values (${placeholder})"; + String sqlTmp = "replace into " + tableName + " (${fields}) values (${placeholder})"; String fieldsStr = ""; String placeholder = ""; diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java index 18d9ba045..b6779529e 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllReqRow.java @@ -24,7 +24,7 @@ import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Maps; +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +43,7 @@ public class OracleAllReqRow extends RdbAllReqRow { private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; public OracleAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(new OracleAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override diff --git a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java index 914a57aef..4063bf9bb 100644 --- a/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java +++ b/oracle/oracle-side/oracle-all-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAllSideInfo.java @@ -21,9 +21,6 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; -import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; import java.util.List; @@ -33,28 +30,4 @@ public class OracleAllSideInfo extends RdbAllSideInfo { public OracleAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } - - @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { - RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; - - sqlCondition = "select ${selectField} from ${tableName} "; - - - sqlCondition = sqlCondition.replace("${tableName}", DtStringUtil.getTableFullPath(rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName())).replace("${selectField}", dealLowerSelectFiled(sideSelectFields)); - System.out.println("---------side_exe_sql-----\n" + sqlCondition); - } - - - private String dealLowerSelectFiled(String fieldsStr) { - StringBuilder sb = new StringBuilder(); - String[] fields = fieldsStr.split(","); - - for(String f : fields) { - sb.append("\"").append(f).append("\"").append(","); - } - - sb.deleteCharAt(sb.lastIndexOf(",")); - return sb.toString(); - } } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java index f973ffee2..c68850d21 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncReqRow.java @@ -43,31 +43,24 @@ public class OracleAsyncReqRow extends RdbAsyncReqRow { private static final String ORACLE_DRIVER = "oracle.jdbc.driver.OracleDriver"; public OracleAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(new OracleAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override public void open(Configuration parameters) throws Exception { super.open(parameters); - JsonObject oracleClientConfig = new JsonObject(); + JsonObject sqlserverClientConfig = new JsonObject(); RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - oracleClientConfig.put("url", rdbSideTableInfo.getUrl()) + sqlserverClientConfig.put("url", rdbSideTableInfo.getUrl()) .put("driver_class", ORACLE_DRIVER) .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) - .put("password", rdbSideTableInfo.getPassword()) - .put("provider_class", DT_PROVIDER_CLASS) - .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) - .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) - .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); - - System.setProperty("vertx.disableFileCPResolving", "true"); + .put("password", rdbSideTableInfo.getPassword()); VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); - vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, oracleClientConfig)); + setRdbSQLClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); } } diff --git a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java index c9d2f98d9..7ff63f3d9 100644 --- a/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java +++ b/oracle/oracle-side/oracle-async-side/src/main/java/com/dtstack/flink/sql/side/oracle/OracleAsyncSideInfo.java @@ -22,12 +22,7 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; -import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import com.dtstack.flink.sql.util.DtStringUtil; -import com.dtstack.flink.sql.util.ParseUtils; -import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; import java.util.List; @@ -37,55 +32,4 @@ public class OracleAsyncSideInfo extends RdbAsyncSideInfo { public OracleAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } - - @Override - public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { - RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideTableInfo; - - String sideTableName = joinInfo.getSideTableName(); - - SqlNode conditionNode = joinInfo.getCondition(); - - List sqlNodeList = Lists.newArrayList(); - List sqlJoinCompareOperate= Lists.newArrayList(); - - ParseUtils.parseAnd(conditionNode, sqlNodeList); - ParseUtils.parseJoinCompareOperate(conditionNode, sqlJoinCompareOperate); - - - for (SqlNode sqlNode : sqlNodeList) { - dealOneEqualCon(sqlNode, sideTableName); - } - - sqlCondition = "select ${selectField} from ${tableName} where "; - for (int i = 0; i < equalFieldList.size(); i++) { - String equalField = sideTableInfo.getPhysicalFields().getOrDefault(equalFieldList.get(i), equalFieldList.get(i)); - sqlCondition += dealLowerFiled(equalField) + " " + sqlJoinCompareOperate.get(i) + " " + " ?"; - if (i != equalFieldList.size() - 1) { - sqlCondition += " and "; - } - } - - sqlCondition = sqlCondition.replace("${tableName}", DtStringUtil.getTableFullPath(rdbSideTableInfo.getSchema(), rdbSideTableInfo.getTableName())).replace("${selectField}", dealLowerSelectFiled(sideSelectFields)); - System.out.println("---------side_exe_sql-----\n" + sqlCondition); - } - - - - private String dealLowerFiled(String field) { - return "\"" + field + "\""; - } - - private String dealLowerSelectFiled(String fieldsStr) { - StringBuilder sb = new StringBuilder(); - String[] fields = fieldsStr.split(","); - - for(String f : fields) { - sb.append("\"").append(f).append("\"").append(","); - } - - sb.deleteCharAt(sb.lastIndexOf(",")); - return sb.toString(); - } - } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java index c30dc9c60..d8669b11f 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/OracleSink.java @@ -21,9 +21,8 @@ import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.ExtendOutputFormat; import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; -import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; -import com.google.common.collect.Lists; +import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; import java.util.ArrayList; import java.util.Iterator; @@ -51,18 +50,17 @@ public RetractJDBCOutputFormat getOutputFormat() { } @Override - public void buildSql(String scheam, String tableName, List fields) { - buildInsertSql(scheam, tableName, fields); + public void buildSql(String tableName, List fields) { + buildInsertSql(tableName, fields); } - private void buildInsertSql(String scheam, String tableName, List fields) { - - tableName = DtStringUtil.getTableFullPath(scheam,tableName); + private void buildInsertSql(String tableName, List fields) { + tableName = quoteTable(tableName); String sqlTmp = "insert into " + tableName + " (${fields}) values (${placeholder})"; List adaptFields = Lists.newArrayList(); - fields.forEach(field -> adaptFields.add(DtStringUtil.addQuoteForStr(field))); + fields.forEach(field -> adaptFields.add(quoteColumn(field))); String fieldsStr = StringUtils.join(adaptFields, ","); String placeholder = ""; @@ -75,37 +73,15 @@ private void buildInsertSql(String scheam, String tableName, List fields this.sql = sqlTmp; } - /** - * use MERGE INTO build oracle replace into sql - * @param tableName - * @param fieldNames create table contained column columns - * @param realIndexes - * @param fullField real columns , query from db - * @return - */ @Override - public String buildUpdateSql(String scheam, String tableName, List fieldNames, Map> realIndexes, List fullField) { - tableName = DtStringUtil.getTableFullPath(scheam, tableName); - - StringBuilder sb = new StringBuilder(); - - sb.append("MERGE INTO " + tableName + " T1 USING " + public String buildUpdateSql(String tableName, List fieldNames, Map> realIndexes, List fullField) { + tableName = quoteTable(tableName); + return "MERGE INTO " + tableName + " T1 USING " + "(" + makeValues(fieldNames) + ") T2 ON (" - + updateKeySql(realIndexes) + ") "); - - - String updateSql = getUpdateSql(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)); - - if (StringUtils.isNotEmpty(updateSql)) { - sb.append(" WHEN MATCHED THEN UPDATE SET "); - sb.append(updateSql); - } - - sb.append(" WHEN NOT MATCHED THEN " + + updateKeySql(realIndexes) + ") WHEN MATCHED THEN UPDATE SET " + + getUpdateSql(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)) + " WHEN NOT MATCHED THEN " + "INSERT (" + quoteColumns(fieldNames) + ") VALUES (" - + quoteColumns(fieldNames, "T2") + ")"); - - return sb.toString(); + + quoteColumns(fieldNames, "T2") + ")"; } @@ -114,22 +90,17 @@ public String quoteColumns(List column) { } public String quoteColumns(List column, String table) { - String prefix = StringUtils.isBlank(table) ? "" : DtStringUtil.addQuoteForStr(table) + "."; + String prefix = StringUtils.isBlank(table) ? "" : quoteTable(table) + "."; List list = new ArrayList<>(); for (String col : column) { - list.add(prefix + DtStringUtil.addQuoteForStr(col)); + list.add(prefix + quoteColumn(col)); } return StringUtils.join(list, ","); } - /** - * extract all distinct index column - * @param realIndexes - * @return - */ - protected List keyColList(Map> realIndexes) { + protected List keyColList(Map> updateKey) { List keyCols = new ArrayList<>(); - for (Map.Entry> entry : realIndexes.entrySet()) { + for (Map.Entry> entry : updateKey.entrySet()) { List list = entry.getValue(); for (String col : list) { if (!containsIgnoreCase(keyCols,col)) { @@ -140,64 +111,56 @@ protected List keyColList(Map> realIndexes) { return keyCols; } - /** - * build update sql , such as UPDATE SET "T1".A="T2".A - * @param updateColumn create table contained column columns - * @param fullColumn real columns , query from db - * @param leftTable alias - * @param rightTable alias - * @param indexCols index column - * @return - */ - public String getUpdateSql(List updateColumn, List fullColumn, String leftTable, String rightTable, List indexCols) { - String prefixLeft = StringUtils.isBlank(leftTable) ? "" : DtStringUtil.addQuoteForStr(leftTable) + "."; - String prefixRight = StringUtils.isBlank(rightTable) ? "" : DtStringUtil.addQuoteForStr(rightTable) + "."; + public String getUpdateSql(List column, List fullColumn, String leftTable, String rightTable, List keyCols) { + String prefixLeft = StringUtils.isBlank(leftTable) ? "" : quoteTable(leftTable) + "."; + String prefixRight = StringUtils.isBlank(rightTable) ? "" : quoteTable(rightTable) + "."; List list = new ArrayList<>(); for (String col : fullColumn) { - // filter index column - if (indexCols == null || indexCols.size() == 0 || containsIgnoreCase(indexCols,col)) { + if (keyCols == null || keyCols.size() == 0 || containsIgnoreCase(keyCols,col)) { continue; } - if (containsIgnoreCase(updateColumn,col)) { - list.add(prefixLeft + DtStringUtil.addQuoteForStr(col) + "=" + prefixRight + DtStringUtil.addQuoteForStr(col)); + if (fullColumn == null ||containsIgnoreCase(column,col)) { + list.add(prefixLeft + col + "=" + prefixRight + col); } else { - list.add(prefixLeft + DtStringUtil.addQuoteForStr(col) + "=null"); + list.add(prefixLeft + col + "=null"); } } return StringUtils.join(list, ","); } + public String quoteTable(String table) { + String[] parts = table.split("\\."); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < parts.length; ++i) { + if (i != 0) { + sb.append("."); + } + sb.append(getStartQuote() + parts[i] + getEndQuote()); + } + return sb.toString(); + } + - /** - * build connect sql by index column, such as T1."A"=T2."A" - * @param updateKey - * @return - */ public String updateKeySql(Map> updateKey) { List exprList = new ArrayList<>(); for (Map.Entry> entry : updateKey.entrySet()) { List colList = new ArrayList<>(); for (String col : entry.getValue()) { - colList.add("T1." + DtStringUtil.addQuoteForStr(col) + "=T2." + DtStringUtil.addQuoteForStr(col)); + colList.add("T1." + quoteColumn(col) + "=T2." + quoteColumn(col)); } exprList.add(StringUtils.join(colList, " AND ")); } return StringUtils.join(exprList, " OR "); } - /** - * build select sql , such as (SELECT ? "A",? "B" FROM DUAL) - * - * @param column destination column - * @return - */ + public String makeValues(List column) { StringBuilder sb = new StringBuilder("SELECT "); for (int i = 0; i < column.size(); ++i) { if (i != 0) { sb.append(","); } - sb.append("? " + DtStringUtil.addQuoteForStr(column.get(i))); + sb.append("? " + quoteColumn(column.get(i))); } sb.append(" FROM DUAL"); return sb.toString(); @@ -212,6 +175,17 @@ public boolean containsIgnoreCase(List l, String s) { return false; } + public String quoteColumn(String column) { + return getStartQuote() + column + getEndQuote(); + } + + public String getStartQuote() { + return "\""; + } + + public String getEndQuote() { + return "\""; + } } diff --git a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java index aff096bd3..6db2c9c06 100644 --- a/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java +++ b/oracle/oracle-sink/src/main/java/com/dtstack/flink/sql/sink/oracle/table/OracleSinkParser.java @@ -34,8 +34,8 @@ public class OracleSinkParser extends RdbSinkParser { @Override public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo oracleTableInfo = super.getTableInfo(tableName, fieldsInfo, props); - oracleTableInfo.setType(CURR_TYPE); - return oracleTableInfo; + TableInfo sqlserverTableInfo = super.getTableInfo(tableName, fieldsInfo, props); + sqlserverTableInfo.setType(CURR_TYPE); + return sqlserverTableInfo; } } diff --git a/pom.xml b/pom.xml index b4c3478b3..0bb26671b 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,6 @@ kafka09 kafka10 kafka11 - kafka mysql hbase elasticsearch5 @@ -25,18 +24,13 @@ oracle cassandra kudu - postgresql - - serversocket - console - clickhouse - - UTF-8 - 1.8.1 - + + UTF-8 + 1.5.4 + diff --git a/postgresql/pom.xml b/postgresql/pom.xml deleted file mode 100644 index 53398b87b..000000000 --- a/postgresql/pom.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - flink.sql - com.dtstack.flink - 1.0-SNAPSHOT - ../pom.xml - - 4.0.0 - pom - sql.postgresql - - - postgresql-side - postgresql-sink - - - 1.0-SNAPSHOT - 42.2.2 - - - - - junit - junit - 3.8.1 - test - - - - com.dtstack.flink - sql.core - ${sql.core.version} - provided - - - org.postgresql - postgresql - ${postgresql.version} - - - diff --git a/postgresql/postgresql-side/pom.xml b/postgresql/postgresql-side/pom.xml deleted file mode 100644 index 0e3723ea6..000000000 --- a/postgresql/postgresql-side/pom.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - sql.postgresql - com.dtstack.flink - 1.0-SNAPSHOT - ../pom.xml - - 4.0.0 - - sql.side.postgresql - 1.0-SNAPSHOT - postgresql-side - pom - - - postgresql-side-core - postgresql-async-side - postgresql-all-side - - - - - 1.0-SNAPSHOT - - - - - com.dtstack.flink - sql.side.rdb - ${rdb.side.version} - - - - - diff --git a/postgresql/postgresql-side/postgresql-all-side/pom.xml b/postgresql/postgresql-side/postgresql-all-side/pom.xml deleted file mode 100644 index d076d9300..000000000 --- a/postgresql/postgresql-side/postgresql-all-side/pom.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - - sql.side.postgresql - com.dtstack.flink - 1.0-SNAPSHOT - ../pom.xml - - 4.0.0 - - sql.side.all.postgresql - postgresql-all-side - - jar - - - 1.0-SNAPSHOT - - - - - com.dtstack.flink - sql.side.postgresql.core - ${sql.side.postgresql.core.version} - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 1.4 - - - package - - shade - - - - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - maven-antrun-plugin - 1.2 - - - copy-resources - - package - - run - - - - - - - - - - - - - - - - - - - diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java deleted file mode 100644 index 6d68cfdca..000000000 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllReqRow.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.side.postgresql; - -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; -import com.dtstack.flink.sql.util.DtStringUtil; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.util.List; -import java.util.Map; - -/** - * side operator with cache for all(period reload) - * Date: 2019-08-11 - * Company: mmg - * - * @author tcm - */ - -public class PostgresqlAllReqRow extends RdbAllReqRow { - - private static final long serialVersionUID = 2098635140857937717L; - - private static final Logger LOG = LoggerFactory.getLogger(PostgresqlAllReqRow.class); - - private static final String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - - public PostgresqlAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(new PostgresqlAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); - } - - @Override - public Connection getConn(String dbURL, String userName, String password) { - try { - Class.forName(POSTGRESQL_DRIVER); - //add param useCursorFetch=true - Map addParams = Maps.newHashMap(); - addParams.put("useCursorFetch", "true"); - String targetDbUrl = DtStringUtil.addJdbcParam(dbURL, addParams, true); - return DriverManager.getConnection(targetDbUrl, userName, password); - } catch (Exception e) { - LOG.error("", e); - throw new RuntimeException("", e); - } - } -} diff --git a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java b/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java deleted file mode 100644 index d383ee46d..000000000 --- a/postgresql/postgresql-side/postgresql-all-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAllSideInfo.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.side.postgresql; - -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo; -import org.apache.flink.api.java.typeutils.RowTypeInfo; - -import java.util.List; - -/** - * Reason: - * Date: 2019-08-11 - * Company: mmg - * - * @author tcm - */ - -public class PostgresqlAllSideInfo extends RdbAllSideInfo { - public PostgresqlAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); - } -} diff --git a/postgresql/postgresql-side/postgresql-async-side/pom.xml b/postgresql/postgresql-side/postgresql-async-side/pom.xml deleted file mode 100644 index 13296e7f3..000000000 --- a/postgresql/postgresql-side/postgresql-async-side/pom.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - - sql.side.postgresql - com.dtstack.flink - 1.0-SNAPSHOT - ../pom.xml - - 4.0.0 - - sql.side.async.postgresql - - postgresql-async-side - - jar - - - 1.0-SNAPSHOT - - - - - com.dtstack.flink - sql.side.postgresql.core - ${sql.side.postgresql.core.version} - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 1.4 - - - package - - shade - - - - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - maven-antrun-plugin - 1.2 - - - copy-resources - - package - - run - - - - - - - - - - - - - - - - - - diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java deleted file mode 100644 index 02b333819..000000000 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncReqRow.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.side.postgresql; - -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.async.RdbAsyncReqRow; -import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import io.vertx.core.Vertx; -import io.vertx.core.VertxOptions; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.jdbc.JDBCClient; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.configuration.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -/** - * Date: 2019-08-11 - * Company: mmg - * - * @author tcm - */ - -public class PostgresqlAsyncReqRow extends RdbAsyncReqRow { - - private static final Logger LOG = LoggerFactory.getLogger(PostgresqlAsyncReqRow.class); - - private final static String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - - public PostgresqlAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(new PostgresqlAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); - } - - - @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - JsonObject pgClientConfig = new JsonObject(); - RdbSideTableInfo rdbSideTableInfo = (RdbSideTableInfo) sideInfo.getSideTableInfo(); - pgClientConfig.put("url", rdbSideTableInfo.getUrl()) - .put("driver_class", POSTGRESQL_DRIVER) - .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) - .put("user", rdbSideTableInfo.getUserName()) - .put("password", rdbSideTableInfo.getPassword()); - - VertxOptions vo = new VertxOptions(); - vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); - vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); - Vertx vertx = Vertx.vertx(vo); - setRdbSQLClient(JDBCClient.createNonShared(vertx, pgClientConfig)); - } - -} diff --git a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java b/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java deleted file mode 100644 index 1d89f4894..000000000 --- a/postgresql/postgresql-side/postgresql-async-side/src/main/java/com/dtstack/flink/sql/side/postgresql/PostgresqlAsyncSideInfo.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.side.postgresql; - -import com.dtstack.flink.sql.side.FieldInfo; -import com.dtstack.flink.sql.side.JoinInfo; -import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.side.rdb.async.RdbAsyncSideInfo; -import org.apache.flink.api.java.typeutils.RowTypeInfo; - -import java.util.List; - -/** - * Reason: - * Date: 2019-08-11 - * Company: mmg - * - * @author tcm - */ - -public class PostgresqlAsyncSideInfo extends RdbAsyncSideInfo { - - public PostgresqlAsyncSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); - } -} diff --git a/postgresql/postgresql-side/postgresql-side-core/pom.xml b/postgresql/postgresql-side/postgresql-side-core/pom.xml deleted file mode 100644 index 6120767eb..000000000 --- a/postgresql/postgresql-side/postgresql-side-core/pom.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - sql.side.postgresql - com.dtstack.flink - 1.0-SNAPSHOT - ../pom.xml - - 4.0.0 - - sql.side.postgresql.core - 1.0-SNAPSHOT - jar - postgresql-side-core - - diff --git a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java b/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java deleted file mode 100644 index faee2c704..000000000 --- a/postgresql/postgresql-side/postgresql-side-core/src/main/java/com/dtstack/flink/sql/side/postgresql/table/PostgresqlSideParser.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.side.postgresql.table; - -import com.dtstack.flink.sql.side.rdb.table.RdbSideParser; -import com.dtstack.flink.sql.table.TableInfo; - -import java.util.Map; - -/** - * Reason: - * Date: 2019-08-11 - * Company: mmg - * - * @author tcm - */ - -public class PostgresqlSideParser extends RdbSideParser { - - private static final String CURR_TYPE = "postgresql"; - - @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - TableInfo pgTableInfo = super.getTableInfo(tableName, fieldsInfo, props); - pgTableInfo.setType(CURR_TYPE); - return pgTableInfo; - } -} diff --git a/postgresql/postgresql-sink/pom.xml b/postgresql/postgresql-sink/pom.xml deleted file mode 100644 index d65368358..000000000 --- a/postgresql/postgresql-sink/pom.xml +++ /dev/null @@ -1,90 +0,0 @@ - - - sql.postgresql - com.dtstack.flink - 1.0-SNAPSHOT - ../pom.xml - - 4.0.0 - - sql.sink.postgresql - jar - - postgresql-sink - http://maven.apache.org - - - 1.0-SNAPSHOT - - - - - com.dtstack.flink - sql.sink.rdb - ${sql.sink.rdb.version} - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 1.4 - - - package - - shade - - - - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - maven-antrun-plugin - 1.2 - - - copy-resources - - package - - run - - - - - - - - - - - - - - - - - - diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java deleted file mode 100644 index 663816ca0..000000000 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/PostgresqlSink.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.sink.postgresql; - - -import com.dtstack.flink.sql.sink.IStreamSinkGener; -import com.dtstack.flink.sql.sink.postgresql.table.PostgresqlTableInfo; -import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; -import com.dtstack.flink.sql.table.TargetTableInfo; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.Map; - -/** - * Date: 2019-08-22 - * Company: mmg - * - * @author tcm - */ - -public class PostgresqlSink extends RdbSink implements IStreamSinkGener { - - private static final String POSTGRESQL_DRIVER = "org.postgresql.Driver"; - - private boolean isUpsert; - - private String keyField; - - public PostgresqlSink() { - } - - @Override - public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { - PostgresqlTableInfo pgTableInfo = (PostgresqlTableInfo) targetTableInfo; - this.isUpsert = pgTableInfo.isUpsert(); - this.keyField = pgTableInfo.getKeyField(); - super.genStreamSink(targetTableInfo); - return this; - } - - @Override - public RetractJDBCOutputFormat getOutputFormat() { - return new RetractJDBCOutputFormat(); - } - - @Override - public void buildSql(String scheam, String tableName, List fields) { - buildInsertSql(tableName, fields); - } - - @Override - public String buildUpdateSql(String schema, String tableName, List fieldNames, Map> realIndexes, List fullField) { - return null; - } - - private void buildInsertSql(String tableName, List fields) { - StringBuffer sqlBuffer = new StringBuffer(); - - sqlBuffer.append("insert into ".concat(tableName) - .concat(" (") - .concat(StringUtils.join(fields, ",")) - .concat(") ") - ); - sqlBuffer.append("values ("); - StringBuffer upsertFields = new StringBuffer(); - for (String fieldName : fields) { - sqlBuffer.append("?,"); - if (this.isUpsert) { - if (fieldName.equals(this.keyField)) { - continue; - } - upsertFields.append(String.format("%s=excluded.%s,", fieldName, fieldName)); - } - } - sqlBuffer.deleteCharAt(sqlBuffer.length() - 1); - sqlBuffer.append(")"); - - if (this.isUpsert) { - upsertFields.deleteCharAt(upsertFields.length() - 1); - sqlBuffer.append(" ON conflict(".concat(keyField).concat(")")); - sqlBuffer.append(" DO UPDATE SET "); - sqlBuffer.append(upsertFields); - } - this.sql = sqlBuffer.toString(); - } - - @Override - public String getDriverName() { - return POSTGRESQL_DRIVER; - } - -} diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java deleted file mode 100644 index f773b5a5c..000000000 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlSinkParser.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.sink.postgresql.table; - -import com.dtstack.flink.sql.sink.rdb.table.RdbSinkParser; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.util.MathUtil; -import org.apache.commons.lang3.StringUtils; - -import java.util.Map; - -/** - * Date: 2019-08-22 - * Company: mmg - * - * @author tcm - */ - -public class PostgresqlSinkParser extends RdbSinkParser { - private static final String CURR_TYPE = "postgresql"; - - @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - PostgresqlTableInfo pgTableInfo = new PostgresqlTableInfo(); - pgTableInfo.setName(tableName); - parseFieldsInfo(fieldsInfo, pgTableInfo); - - pgTableInfo.setParallelism(MathUtil.getIntegerVal(props.get(PostgresqlTableInfo.PARALLELISM_KEY.toLowerCase()))); - pgTableInfo.setUrl(MathUtil.getString(props.get(PostgresqlTableInfo.URL_KEY.toLowerCase()))); - pgTableInfo.setTableName(MathUtil.getString(props.get(PostgresqlTableInfo.TABLE_NAME_KEY.toLowerCase()))); - pgTableInfo.setUserName(MathUtil.getString(props.get(PostgresqlTableInfo.USER_NAME_KEY.toLowerCase()))); - pgTableInfo.setPassword(MathUtil.getString(props.get(PostgresqlTableInfo.PASSWORD_KEY.toLowerCase()))); - pgTableInfo.setBatchSize(MathUtil.getIntegerVal(props.get(PostgresqlTableInfo.BATCH_SIZE_KEY.toLowerCase()))); - pgTableInfo.setBatchWaitInterval(MathUtil.getLongVal(props.get(PostgresqlTableInfo.BATCH_WAIT_INTERVAL_KEY.toLowerCase()))); - pgTableInfo.setBufferSize(MathUtil.getString(props.get(PostgresqlTableInfo.BUFFER_SIZE_KEY.toLowerCase()))); - pgTableInfo.setFlushIntervalMs(MathUtil.getString(props.get(PostgresqlTableInfo.FLUSH_INTERVALMS_KEY.toLowerCase()))); - - pgTableInfo.setKeyField(MathUtil.getString(props.get(PostgresqlTableInfo.TABLE_KEY_FIELD.toLowerCase()))); - - String isUpsertStr = (String) props.get(PostgresqlTableInfo.TABLE_IS_UPSERT.toLowerCase()); - pgTableInfo.setUpsert(!StringUtils.isEmpty(isUpsertStr) && isUpsertStr.equals("true") ? true : false); - - pgTableInfo.check(); - return pgTableInfo; - } -} diff --git a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlTableInfo.java b/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlTableInfo.java deleted file mode 100644 index 78df0de14..000000000 --- a/postgresql/postgresql-sink/src/main/java/com/dtstack/flink/sql/sink/postgresql/table/PostgresqlTableInfo.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.dtstack.flink.sql.sink.postgresql.table; - -import com.dtstack.flink.sql.sink.rdb.table.RdbTableInfo; -import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; - -/** - * Date: 2019-08-22 - * Company: mmg - * - * @author tcm - */ - -public class PostgresqlTableInfo extends RdbTableInfo { - - public static final String TABLE_IS_UPSERT = "isUpsert"; - - public static final String TABLE_KEY_FIELD = "keyField"; - - private static final String CURR_TYPE = "postgresql"; - - private boolean isUpsert; - - private String keyField; - - - public PostgresqlTableInfo() { - setType(CURR_TYPE); - } - - public boolean isUpsert() { - return isUpsert; - } - - public void setUpsert(boolean upsert) { - isUpsert = upsert; - } - - public String getKeyField() { - return keyField; - } - - public void setKeyField(String keyField) { - this.keyField = keyField; - } - - @Override - public boolean check() { - Preconditions.checkNotNull(getUrl(), "postgresql field of URL is required"); - Preconditions.checkNotNull(getTableName(), "postgresql field of tableName is required"); - Preconditions.checkNotNull(getUserName(), "postgresql field of userName is required"); - Preconditions.checkNotNull(getPassword(), "postgresql field of password is required"); - if (isUpsert()) { - Preconditions.checkNotNull(getKeyField(), "postgresql field of keyField is required"); - } - return true; - } -} diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java index 5fc7ba158..abae0cbb2 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllReqRow.java @@ -19,24 +19,26 @@ package com.dtstack.flink.sql.side.rdb.all; import com.dtstack.flink.sql.side.AllReqRow; -import com.dtstack.flink.sql.side.SideInfo; +import com.dtstack.flink.sql.side.FieldInfo; +import com.dtstack.flink.sql.side.JoinInfo; +import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; +import com.dtstack.flink.sql.side.rdb.util.MathUtil; import com.dtstack.flink.sql.side.rdb.util.SwitchUtil; import org.apache.calcite.sql.JoinType; import org.apache.commons.collections.CollectionUtils; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Maps; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; import org.apache.flink.util.Collector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Timestamp; +import java.math.BigDecimal; +import java.sql.*; import java.util.Calendar; import java.util.List; import java.util.Map; @@ -60,8 +62,9 @@ public abstract class RdbAllReqRow extends AllReqRow { private AtomicReference>>> cacheRef = new AtomicReference<>(); - public RdbAllReqRow(SideInfo sideInfo) { - super(sideInfo); + + public RdbAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + super(new RdbAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override @@ -119,12 +122,9 @@ public void flatMap(Row value, Collector out) throws Exception { for (Integer conValIndex : sideInfo.getEqualValIndex()) { Object equalObj = value.getField(conValIndex); if (equalObj == null) { - if (sideInfo.getJoinType() == JoinType.LEFT) { - Row row = fillData(value, null); - out.collect(row); - } - return; + out.collect(null); } + inputParams.add(equalObj); } @@ -174,6 +174,7 @@ private void loadData(Map>> tmpCache) throws SQ try { for (int i = 0; i < CONN_RETRY_NUM; i++) { + try { connection = getConn(tableInfo.getUrl(), tableInfo.getUserName(), tableInfo.getPassword()); break; @@ -181,6 +182,7 @@ private void loadData(Map>> tmpCache) throws SQ if (i == CONN_RETRY_NUM - 1) { throw new RuntimeException("", e); } + try { String connInfo = "url:" + tableInfo.getUrl() + ";userName:" + tableInfo.getUserName() + ",pwd:" + tableInfo.getPassword(); LOG.warn("get conn fail, wait for 5 sec and try again, connInfo:" + connInfo); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java index c10b394e5..501686f95 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/all/RdbAllSideInfo.java @@ -23,11 +23,12 @@ import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import com.dtstack.flink.sql.util.ParseUtils; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.commons.collections.CollectionUtils; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import java.util.List; @@ -54,6 +55,7 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { sqlCondition = "select ${selectField} from ${tableName} "; sqlCondition = sqlCondition.replace("${tableName}", rdbSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); + System.out.println("---------side_exe_sql-----\n" + sqlCondition); } @Override @@ -86,8 +88,11 @@ public void parseSelectFields(JoinInfo joinInfo) { SqlNode conditionNode = joinInfo.getCondition(); List sqlNodeList = Lists.newArrayList(); - - ParseUtils.parseAnd(conditionNode, sqlNodeList); + if (conditionNode.getKind() == SqlKind.AND) { + sqlNodeList.addAll(Lists.newArrayList(((SqlBasicCall) conditionNode).getOperands())); + } else { + sqlNodeList.add(conditionNode); + } for (SqlNode sqlNode : sqlNodeList) { dealOneEqualCon(sqlNode, sideTableName); diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java index d997f7d58..6989dd64f 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java @@ -26,7 +26,8 @@ import io.vertx.core.json.JsonArray; import io.vertx.ext.sql.SQLClient; import io.vertx.ext.sql.SQLConnection; -import com.google.common.collect.Lists; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; @@ -56,18 +57,10 @@ public class RdbAsyncReqRow extends AsyncReqRow { public final static int DEFAULT_MAX_DB_CONN_POOL_SIZE = DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE + DEFAULT_VERTX_WORKER_POOL_SIZE; - public final static int DEFAULT_IDLE_CONNECTION_TEST_PEROID = 60; - - public final static boolean DEFAULT_TEST_CONNECTION_ON_CHECKIN = true; - - public final static String DT_PROVIDER_CLASS = "com.dtstack.flink.sql.side.rdb.provider.DTC3P0DataSourceProvider"; - - public final static String PREFERRED_TEST_QUERY_SQL = "select 1 from dual"; - private transient SQLClient rdbSQLClient; - public RdbAsyncReqRow(SideInfo sideInfo) { - super(sideInfo); + public RdbAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { + super(new RdbAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); } @Override @@ -77,7 +70,7 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except for (Integer conValIndex : sideInfo.getEqualValIndex()) { Object equalObj = input.getField(conValIndex); if (equalObj == null) { - dealMissKey(input, resultFuture); + resultFuture.complete(null); return; } inputParams.add(equalObj); @@ -99,7 +92,7 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except } resultFuture.complete(rowList); } else { - resultFuture.completeExceptionally(new RuntimeException("not support cache obj type " + val.getType())); + throw new RuntimeException("not support cache obj type " + val.getType()); } return; } @@ -117,7 +110,7 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except connection.queryWithParams(sqlCondition, inputParams, rs -> { if (rs.failed()) { LOG.error("Cannot retrieve the data from the database", rs.cause()); - resultFuture.completeExceptionally(rs.cause()); + resultFuture.complete(null); return; } diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java index 0481e3443..f652bf342 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncSideInfo.java @@ -23,19 +23,14 @@ import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.SideTableInfo; import com.dtstack.flink.sql.side.rdb.table.RdbSideTableInfo; -import com.dtstack.flink.sql.util.ParseUtils; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.*; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; -import java.util.Arrays; import java.util.List; -import java.util.Map; - /** * Reason: @@ -62,56 +57,35 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { SqlNode conditionNode = joinInfo.getCondition(); List sqlNodeList = Lists.newArrayList(); - - List sqlJoinCompareOperate= Lists.newArrayList(); - - ParseUtils.parseAnd(conditionNode, sqlNodeList); - ParseUtils.parseJoinCompareOperate(conditionNode, sqlJoinCompareOperate); + if (conditionNode.getKind() == SqlKind.AND) { + sqlNodeList.addAll(Lists.newArrayList(((SqlBasicCall) conditionNode).getOperands())); + } else { + sqlNodeList.add(conditionNode); + } for (SqlNode sqlNode : sqlNodeList) { dealOneEqualCon(sqlNode, sideTableName); } - List whereConditionList = Lists.newArrayList();; - Map physicalFields = rdbSideTableInfo.getPhysicalFields(); - SqlNode whereNode = ((SqlSelect) joinInfo.getSelectNode()).getWhere(); - if (whereNode != null) { - // 解析维表中的过滤条件 - ParseUtils.parseSideWhere(whereNode, physicalFields, whereConditionList); - } - sqlCondition = "select ${selectField} from ${tableName} where "; for (int i = 0; i < equalFieldList.size(); i++) { - String equalField = sideTableInfo.getPhysicalFields().getOrDefault(equalFieldList.get(i), equalFieldList.get(i)); + String equalField = equalFieldList.get(i); - sqlCondition += equalField + " " + sqlJoinCompareOperate.get(i) + " ? "; + sqlCondition += equalField + "=? "; if (i != equalFieldList.size() - 1) { sqlCondition += " and "; } } - if (0 != whereConditionList.size()) { - // 如果where条件中第一个符合条件的是维表中的条件 - String firstCondition = whereConditionList.get(0); - if (!"and".equalsIgnoreCase(firstCondition) && !"or".equalsIgnoreCase(firstCondition)) { - whereConditionList.add(0, "and ("); - } else { - whereConditionList.add(1, "("); - } - whereConditionList.add(whereConditionList.size(), ")"); - sqlCondition += String.join(" ", whereConditionList); - } sqlCondition = sqlCondition.replace("${tableName}", rdbSideTableInfo.getTableName()).replace("${selectField}", sideSelectFields); - - System.out.println("--------side sql query:-------------------"); - System.out.println(sqlCondition); + System.out.println("---------side_exe_sql-----\n" + sqlCondition); } @Override public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { - if (!SqlKind.COMPARISON.contains(sqlNode.getKind())) { - throw new RuntimeException("not compare operator."); + if (sqlNode.getKind() != SqlKind.EQUALS) { + throw new RuntimeException("not equal operator."); } SqlIdentifier left = (SqlIdentifier) ((SqlBasicCall) sqlNode).getOperands()[0]; @@ -160,4 +134,4 @@ public void dealOneEqualCon(SqlNode sqlNode, String sideTableName) { } -} \ No newline at end of file +} diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java deleted file mode 100644 index fcca4f0ef..000000000 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/provider/DTC3P0DataSourceProvider.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.dtstack.flink.sql.side.rdb.provider; - -import com.mchange.v2.c3p0.ComboPooledDataSource; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.jdbc.spi.impl.C3P0DataSourceProvider; - -import javax.sql.DataSource; -import java.beans.PropertyVetoException; -import java.sql.SQLException; - -public class DTC3P0DataSourceProvider extends C3P0DataSourceProvider { - - @Override - public DataSource getDataSource(JsonObject config) throws SQLException { - String url = config.getString("url"); - if (url == null) throw new NullPointerException("url cannot be null"); - String driverClass = config.getString("driver_class"); - String user = config.getString("user"); - String password = config.getString("password"); - Integer maxPoolSize = config.getInteger("max_pool_size"); - Integer initialPoolSize = config.getInteger("initial_pool_size"); - Integer minPoolSize = config.getInteger("min_pool_size"); - Integer maxStatements = config.getInteger("max_statements"); - Integer maxStatementsPerConnection = config.getInteger("max_statements_per_connection"); - Integer maxIdleTime = config.getInteger("max_idle_time"); - Integer acquireRetryAttempts = config.getInteger("acquire_retry_attempts"); - Integer acquireRetryDelay = config.getInteger("acquire_retry_delay"); - Boolean breakAfterAcquireFailure = config.getBoolean("break_after_acquire_failure"); - - //add c3p0 params - String preferredTestQuery = config.getString("preferred_test_query"); - Integer idleConnectionTestPeriod = config.getInteger("idle_connection_test_period"); - Boolean testConnectionOnCheckin = config.getBoolean("test_connection_on_checkin"); - - - - // If you want to configure any other C3P0 properties you can add a file c3p0.properties to the classpath - ComboPooledDataSource cpds = new ComboPooledDataSource(); - cpds.setJdbcUrl(url); - if (driverClass != null) { - try { - cpds.setDriverClass(driverClass); - } catch (PropertyVetoException e) { - throw new IllegalArgumentException(e); - } - } - if (user != null) { - cpds.setUser(user); - } - if (password != null) { - cpds.setPassword(password); - } - if (maxPoolSize != null) { - cpds.setMaxPoolSize(maxPoolSize); - } - if (minPoolSize != null) { - cpds.setMinPoolSize(minPoolSize); - } - if (initialPoolSize != null) { - cpds.setInitialPoolSize(initialPoolSize); - } - if (maxStatements != null) { - cpds.setMaxStatements(maxStatements); - } - if (maxStatementsPerConnection != null) { - cpds.setMaxStatementsPerConnection(maxStatementsPerConnection); - } - if (maxIdleTime != null) { - cpds.setMaxIdleTime(maxIdleTime); - } - if(acquireRetryAttempts != null){ - cpds.setAcquireRetryAttempts(acquireRetryAttempts); - } - if(acquireRetryDelay != null){ - cpds.setAcquireRetryDelay(acquireRetryDelay); - } - if(breakAfterAcquireFailure != null){ - cpds.setBreakAfterAcquireFailure(breakAfterAcquireFailure); - } - - if (preferredTestQuery != null) { - cpds.setPreferredTestQuery(preferredTestQuery); - } - - if (idleConnectionTestPeriod != null) { - cpds.setIdleConnectionTestPeriod(idleConnectionTestPeriod); - } - - if (testConnectionOnCheckin != null) { - cpds.setTestConnectionOnCheckin(testConnectionOnCheckin); - } - - return cpds; - } -} diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java index b9811b0ee..e007cecb0 100644 --- a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java +++ b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/table/RdbSideParser.java @@ -47,8 +47,6 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map, Serializab protected String dbType; - protected int batchNum = 100; - - protected long batchWaitInterval = 10000; + protected int batchInterval = 1; protected int[] sqlTypes; protected String tableName; - protected String registerTabName; - protected String sql; protected List primaryKeys; @@ -81,8 +77,6 @@ public abstract class RdbSink implements RetractStreamTableSink, Serializab private int parallelism = -1; - private String schema; - public RichSinkFunction createJdbcSinkFunc() { if (driverName == null || dbURL == null || userName == null || password == null || sqlTypes == null || tableName == null) { @@ -95,12 +89,10 @@ public RichSinkFunction createJdbcSinkFunc() { outputFormat.setUsername(userName); outputFormat.setPassword(password); outputFormat.setInsertQuery(sql); - outputFormat.setBatchNum(batchNum); - outputFormat.setBatchWaitInterval(batchWaitInterval); + outputFormat.setBatchInterval(batchInterval); outputFormat.setTypesArray(sqlTypes); outputFormat.setTableName(tableName); outputFormat.setDbType(dbType); - outputFormat.setSchema(schema); outputFormat.setDbSink(this); outputFormat.verifyField(); @@ -117,16 +109,10 @@ public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { String tmpUserName = rdbTableInfo.getUserName(); String tmpPassword = rdbTableInfo.getPassword(); String tmpTableName = rdbTableInfo.getTableName(); - String tmpRegisterName = rdbTableInfo.getName(); Integer tmpSqlBatchSize = rdbTableInfo.getBatchSize(); if (tmpSqlBatchSize != null) { - setBatchNum(tmpSqlBatchSize); - } - - Long batchWaitInterval = rdbTableInfo.getBatchWaitInterval(); - if (batchWaitInterval != null) { - setBatchWaitInterval(batchWaitInterval); + setBatchInterval(tmpSqlBatchSize); } Integer tmpSinkParallelism = rdbTableInfo.getParallelism(); @@ -142,12 +128,10 @@ public RdbSink genStreamSink(TargetTableInfo targetTableInfo) { this.userName = tmpUserName; this.password = tmpPassword; this.tableName = tmpTableName; - this.registerTabName = tmpRegisterName; this.primaryKeys = rdbTableInfo.getPrimaryKeys(); this.dbType = rdbTableInfo.getType(); - this.schema = rdbTableInfo.getSchema(); - buildSql(schema, tableName, fields); + buildSql(tableName, fields); buildSqlTypes(fieldTypeArray); return this; } @@ -201,7 +185,7 @@ protected void buildSqlTypes(List fieldTypeArray) { public void emitDataStream(DataStream> dataStream) { RichSinkFunction richSinkFunction = createJdbcSinkFunc(); DataStreamSink streamSink = dataStream.addSink(richSinkFunction); - streamSink.name(registerTabName); + streamSink.name(tableName); if (parallelism > 0) { streamSink.setParallelism(parallelism); } @@ -214,13 +198,13 @@ public TableSink> configure(String[] fieldNames, TypeInform return this; } - - public void setBatchNum(int batchNum) { - this.batchNum = batchNum; - } - - public void setBatchWaitInterval(long batchWaitInterval) { - this.batchWaitInterval = batchWaitInterval; + /** + * Set the default frequency submit updated every submission + * + * @param batchInterval + */ + public void setBatchInterval(int batchInterval) { + this.batchInterval = batchInterval; } @Override @@ -258,7 +242,7 @@ public void setDbType(String dbType) { * @param tableName * @param fields */ - public abstract void buildSql(String schema, String tableName, List fields); + public abstract void buildSql(String tableName, List fields); /** * sqlserver and oracle maybe implement @@ -268,7 +252,7 @@ public void setDbType(String dbType) { * @param realIndexes * @return */ - public abstract String buildUpdateSql(String schema, String tableName, List fieldNames, Map> realIndexes, List fullField); + public abstract String buildUpdateSql(String tableName, List fieldNames, Map> realIndexes, List fullField); public abstract String getDriverName(); diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/ExtendOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/ExtendOutputFormat.java index 6265bbadd..93391b713 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/ExtendOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/ExtendOutputFormat.java @@ -18,9 +18,8 @@ package com.dtstack.flink.sql.sink.rdb.format; -import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; -import com.google.common.collect.Maps; +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; import java.sql.ResultSet; import java.sql.SQLException; @@ -63,43 +62,40 @@ public boolean isReplaceInsertQuery() throws SQLException { */ public void fillRealIndexes() throws SQLException { Map> map = Maps.newHashMap(); - - ResultSet rs = getDbConn().getMetaData().getIndexInfo(null, getSchema(), DtStringUtil.addQuoteForStr(getTableName()), true, false); + ResultSet rs = getDbConn().getMetaData().getIndexInfo(null, null, getTableName(), true, true); while (rs.next()) { String indexName = rs.getString("INDEX_NAME"); - if (StringUtils.isNotBlank(indexName) && !map.containsKey(indexName)) { + if (!map.containsKey(indexName)) { map.put(indexName, new ArrayList<>()); } String column_name = rs.getString("COLUMN_NAME"); if (StringUtils.isNotBlank(column_name)) { - map.get(indexName).add(column_name); + column_name = column_name.toUpperCase(); } + map.get(indexName).add(column_name); } for (Map.Entry> entry : map.entrySet()) { String k = entry.getKey(); List v = entry.getValue(); if (v != null && v.size() != 0 && v.get(0) != null) { - realIndexesAdd(k, v); + getRealIndexes().put(k, v); } } } - - /** * get db all column name * * @throws SQLException */ public void fillFullColumns() throws SQLException { - // table name not quote - ResultSet rs = getDbConn().getMetaData().getColumns(null, getSchema(), getTableName(), null); + ResultSet rs = getDbConn().getMetaData().getColumns(null, null, getTableName(), null); while (rs.next()) { String columnName = rs.getString("COLUMN_NAME"); if (StringUtils.isNotBlank(columnName)) { - fullFieldAdd(columnName); + getFullField().add(columnName.toUpperCase()); } } } @@ -112,5 +108,4 @@ public boolean containsIgnoreCase(List l, String s) { } return false; } - } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java index 7653c847c..382b687e1 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/format/RetractJDBCOutputFormat.java @@ -19,67 +19,48 @@ package com.dtstack.flink.sql.sink.rdb.format; import com.dtstack.flink.sql.sink.rdb.RdbSink; -import com.dtstack.flink.sql.util.JDBCUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.java.tuple.Tuple; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.*; +import java.util.*; import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import com.dtstack.flink.sql.sink.MetricOutputFormat; -import sun.rmi.runtime.Log; /** * OutputFormat to write tuples into a database. * The OutputFormat has to be configured using the supplied OutputFormatBuilder. * + * @see Tuple + * @see DriverManager */ public class RetractJDBCOutputFormat extends MetricOutputFormat { private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(RetractJDBCOutputFormat.class); - private static int dirtyDataPrintFrequency = 1000; - - private static int receiveDataPrintFrequency = 1000; - private String username; private String password; private String drivername; private String dbURL; private String tableName; private String dbType; - private String schema; private RdbSink dbSink; - // trigger preparedStatement execute batch interval - private long batchWaitInterval = 10000l; - // PreparedStatement execute batch num - private int batchNum = 100; + private int batchInterval = 5000; private String insertQuery; public int[] typesArray; - /** 存储用于批量写入的数据 */ - protected List rows = new ArrayList(); - private Connection dbConn; private PreparedStatement upload; - private transient ScheduledThreadPoolExecutor timerService; + private int batchCount = 0; //index field private Map> realIndexes = Maps.newHashMap(); @@ -87,6 +68,7 @@ public class RetractJDBCOutputFormat extends MetricOutputFormat { private List fullField = Lists.newArrayList(); public RetractJDBCOutputFormat() { + } @Override @@ -103,49 +85,32 @@ public void configure(Configuration parameters) { @Override public void open(int taskNumber, int numTasks) throws IOException { try { - LOG.info("PreparedStatement execute batch num is {}", batchNum); - dbConn = establishConnection(); + establishConnection(); initMetric(); - - if (existTabname()) { + if (dbConn.getMetaData().getTables(null, null, tableName, null).next()) { if (isReplaceInsertQuery()) { - insertQuery = dbSink.buildUpdateSql(schema , tableName, Arrays.asList(dbSink.getFieldNames()), realIndexes, fullField); + insertQuery = dbSink.buildUpdateSql(tableName, Arrays.asList(dbSink.getFieldNames()), realIndexes, fullField); } upload = dbConn.prepareStatement(insertQuery); } else { throw new SQLException("Table " + tableName + " doesn't exist"); } - if (batchWaitInterval > 0 && batchNum > 1) { - LOG.info("open batch wait interval scheduled, interval is {} ms", batchWaitInterval); - - timerService = new ScheduledThreadPoolExecutor(1); - timerService.scheduleAtFixedRate(() -> { - submitExecuteBatch(); - }, 0, batchWaitInterval, TimeUnit.MILLISECONDS); - - } - } catch (SQLException sqe) { - LOG.error("", sqe); throw new IllegalArgumentException("open() failed.", sqe); } catch (ClassNotFoundException cnfe) { - LOG.error("", cnfe); throw new IllegalArgumentException("JDBC driver class not found.", cnfe); } } - private Connection establishConnection() throws SQLException, ClassNotFoundException { - Connection connection ; - JDBCUtils.forName(drivername, getClass().getClassLoader()); + private void establishConnection() throws SQLException, ClassNotFoundException { + Class.forName(drivername); if (username == null) { - connection = DriverManager.getConnection(dbURL); + dbConn = DriverManager.getConnection(dbURL); } else { - connection = DriverManager.getConnection(dbURL, username, password); + dbConn = DriverManager.getConnection(dbURL, username, password); } - connection.setAutoCommit(false); - return connection; } /** @@ -162,58 +127,37 @@ private Connection establishConnection() throws SQLException, ClassNotFoundExcep * @see PreparedStatement */ @Override - public void writeRecord(Tuple2 tuple2) { + public void writeRecord(Tuple2 tuple2) throws IOException { Tuple2 tupleTrans = tuple2; Boolean retract = tupleTrans.getField(0); Row row = tupleTrans.getField(1); + if (typesArray != null && typesArray.length > 0 && typesArray.length != row.getArity()) { LOG.warn("Column SQL types array doesn't match arity of passed Row! Check the passed array..."); } - - if (retract) { - outRecords.inc(); - if (outRecords.getCount() % receiveDataPrintFrequency == 0) { - LOG.info("Receive data : {}", row); - } - insertWrite(row); - } else { - //do nothing - } - } - - - private void insertWrite(Row row) { - checkConnectionOpen(dbConn); try { - if (batchNum == 1) { - writeSingleRecord(row); + if (retract) { + insertWrite(row); + outRecords.inc(); } else { - updatePreparedStmt(row, upload); - rows.add(row); - upload.addBatch(); - if (rows.size() >= batchNum) { - submitExecuteBatch(); - } + //do nothing } - } catch (SQLException e) { - LOG.error("", e); + } catch (SQLException | IllegalArgumentException e) { + throw new IllegalArgumentException("writeRecord() failed", e); } - } - private void writeSingleRecord(Row row) { - try { - updatePreparedStmt(row, upload); - upload.executeUpdate(); - dbConn.commit(); - } catch (SQLException e) { - outDirtyRecords.inc(); - if (outDirtyRecords.getCount() % dirtyDataPrintFrequency == 0 || LOG.isDebugEnabled()) { - LOG.error("record insert failed ..", row.toString()); - LOG.error("", e); - } + + private void insertWrite(Row row) throws SQLException { + + updatePreparedStmt(row, upload); + upload.addBatch(); + batchCount++; + if (batchCount >= batchInterval) { + upload.executeBatch(); + batchCount = 0; } } @@ -306,39 +250,6 @@ private void updatePreparedStmt(Row row, PreparedStatement pstmt) throws SQLExce } } - - private synchronized void submitExecuteBatch() { - try { - this.upload.executeBatch(); - dbConn.commit(); - } catch (SQLException e) { - try { - dbConn.rollback(); - } catch (SQLException e1) { - LOG.error("rollback data error !", e); - } - - rows.forEach(this::writeSingleRecord); - } finally { - rows.clear(); - } - } - - private void checkConnectionOpen(Connection dbConn) { - try { - if (dbConn.isClosed()) { - LOG.info("db connection reconnect.."); - dbConn= establishConnection(); - upload = dbConn.prepareStatement(insertQuery); - this.dbConn = dbConn; - } - } catch (SQLException e) { - LOG.error("check connection open failed..", e); - } catch (ClassNotFoundException e) { - LOG.error("load jdbc class error when reconnect db..", e); - } - } - /** * Executes prepared statement and closes all resources of this instance. * @@ -351,15 +262,11 @@ public void close() throws IOException { upload.executeBatch(); upload.close(); } - if (null != timerService) { - timerService.shutdown(); - LOG.info("batch wait interval scheduled service closed "); - } } catch (SQLException se) { - LOG.info("Inputformat couldn't be closed - ", se); + LOG.info("Inputformat couldn't be closed - " + se.getMessage()); } finally { upload = null; - rows.clear(); + batchCount = 0; } try { @@ -367,7 +274,7 @@ public void close() throws IOException { dbConn.close(); } } catch (SQLException se) { - LOG.info("Inputformat couldn't be closed - ", se); + LOG.info("Inputformat couldn't be closed - " + se.getMessage()); } finally { dbConn = null; } @@ -401,21 +308,6 @@ public void setUsername(String username) { this.username = username; } - public String getSchema() { - if (StringUtils.isNotEmpty(schema)) { - return schema; - } - return null; - } - - public void setSchema(String schema) { - this.schema = schema; - } - - public boolean existTabname() throws SQLException { - return dbConn.getMetaData().getTables(null, getSchema(), tableName, null).next(); - } - public void setPassword(String password) { this.password = password; } @@ -440,8 +332,8 @@ public void setDbSink(RdbSink dbSink) { this.dbSink = dbSink; } - public void setBatchNum(int batchNum) { - this.batchNum = batchNum; + public void setBatchInterval(int batchInterval) { + this.batchInterval = batchInterval; } public void setInsertQuery(String insertQuery) { @@ -468,24 +360,11 @@ public String getTableName() { return tableName; } - public void realIndexesAdd(String index, List fieldes) { - this.realIndexes.put(index, fieldes); - } - - public Map> getRealIndexes() { + public Map> getRealIndexes() { return realIndexes; } - - public void setBatchWaitInterval(long batchWaitInterval) { - this.batchWaitInterval = batchWaitInterval; - } - public List getFullField() { return fullField; } - - public void fullFieldAdd(String colName) { - this.fullField.add(colName); - } } diff --git a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java index beb51ffaa..0c248997b 100644 --- a/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java +++ b/rdb/rdb-sink/src/main/java/com/dtstack/flink/sql/sink/rdb/table/RdbSinkParser.java @@ -43,11 +43,8 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map out) throws Exception { out.collect(null); } String columnName = sideInfo.getEqualFieldList().get(conValIndex); - inputParams.put(columnName, equalObj.toString()); + inputParams.put(columnName, (String) equalObj); } String key = buildKey(inputParams); diff --git a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java index 81e7b1f4f..29564a46d 100644 --- a/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java +++ b/redis5/redis5-side/redis-all-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAllSideInfo.java @@ -22,10 +22,11 @@ import com.dtstack.flink.sql.side.JoinInfo; import com.dtstack.flink.sql.side.SideInfo; import com.dtstack.flink.sql.side.SideTableInfo; -import com.dtstack.flink.sql.util.ParseUtils; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; +import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; import java.util.List; @@ -43,8 +44,11 @@ public void buildEqualInfo(JoinInfo joinInfo, SideTableInfo sideTableInfo) { SqlNode conditionNode = joinInfo.getCondition(); List sqlNodeList = Lists.newArrayList(); - - ParseUtils.parseAnd(conditionNode, sqlNodeList); + if(conditionNode.getKind() == SqlKind.AND){ + sqlNodeList.addAll(Lists.newArrayList(((SqlBasicCall)conditionNode).getOperands())); + }else{ + sqlNodeList.add(conditionNode); + } for(SqlNode sqlNode : sqlNodeList){ dealOneEqualCon(sqlNode, sideTableName); diff --git a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java index e2b256421..412fca5cc 100644 --- a/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java +++ b/redis5/redis5-side/redis-async-side/src/main/java/com/dtstack/flink/sql/side/redis/RedisAsyncReqRow.java @@ -32,12 +32,14 @@ import io.lettuce.core.cluster.RedisClusterClient; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import org.apache.flink.calcite.shaded.com.google.common.collect.Lists; +import org.apache.flink.calcite.shaded.com.google.common.collect.Maps; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.types.Row; +import java.sql.Timestamp; import java.util.Collections; import java.util.List; import java.util.Map; @@ -122,12 +124,11 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except Integer conValIndex = sideInfo.getEqualValIndex().get(i); Object equalObj = input.getField(conValIndex); if(equalObj == null){ - dealMissKey(input, resultFuture); - return; + resultFuture.complete(null); } - String value = equalObj.toString(); + keyData.add(sideInfo.getEqualFieldList().get(i)); - keyData.add(value); + keyData.add((String) equalObj); } String key = buildCacheKey(keyData); @@ -147,8 +148,7 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except } resultFuture.complete(rowList); }else{ - RuntimeException exception = new RuntimeException("not support cache obj type " + val.getType()); - resultFuture.completeExceptionally(exception); + throw new RuntimeException("not support cache obj type " + val.getType()); } return; } @@ -156,28 +156,28 @@ public void asyncInvoke(Row input, ResultFuture resultFuture) throws Except Map keyValue = Maps.newHashMap(); List value = async.keys(key + ":*").get(); - String[] values = value.toArray(new String[value.size()]); - if (values.length == 0){ - dealMissKey(input, resultFuture); + if (value.size() == 0){ + dealMissKey(input,null); } else { - RedisFuture>> future = ((RedisStringAsyncCommands) async).mget(values); + String[] values = value.toArray(new String[value.size()]); + RedisFuture>> future = ((RedisStringAsyncCommands) async).mget(values); future.thenAccept(new Consumer>>() { @Override public void accept(List> keyValues) { - if (keyValues.size() != 0) { - for (int i = 0; i < keyValues.size(); i++) { + if (keyValues.size() != 0){ + for (int i=0; i sqlNodeList = Lists.newArrayList(); - ParseUtils.parseAnd(conditionNode, sqlNodeList); + if(conditionNode.getKind() == SqlKind.AND){ + sqlNodeList.addAll(Lists.newArrayList(((SqlBasicCall)conditionNode).getOperands())); + }else{ + sqlNodeList.add(conditionNode); + } for(SqlNode sqlNode : sqlNodeList){ dealOneEqualCon(sqlNode, sideTableName); diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java index 5007868e9..40dae6c68 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideReqRow.java @@ -24,8 +24,6 @@ import org.apache.flink.types.Row; import java.io.Serializable; -import java.math.BigDecimal; -import java.sql.Date; import java.sql.Timestamp; import java.util.Map; @@ -65,50 +63,10 @@ public Row fillData(Row input, Object sideInput) { row.setField(entry.getKey(), null); }else{ String key = sideInfo.getSideFieldNameIndex().get(entry.getKey()); - setRowField(row, entry.getKey(), sideInfo, sideInputMap.get(key)); + row.setField(entry.getKey(), sideInputMap.get(key)); } } return row; } - - public void setRowField(Row row, Integer index, SideInfo sideInfo, String value) { - Integer keyIndex = sideInfo.getSideFieldIndex().get(index); - String classType = sideInfo.getSideTableInfo().getFieldClassList().get(keyIndex).getName(); - switch (classType){ - case "java.lang.Integer": - row.setField(index, Integer.valueOf(value)); - break; - case "java.lang.String": - row.setField(index, value); - break; - case "java.lang.Double": - row.setField(index, Double.valueOf(value)); - break; - case "java.lang.Long": - row.setField(index, Long.valueOf(value)); - break; - case "java.lang.Byte": - row.setField(index, Byte.valueOf(value)); - break; - case "java.lang.Short": - row.setField(index, Short.valueOf(value)); - break; - case "java.lang.Float": - row.setField(index, Float.valueOf(value)); - break; - case "java.math.BigDecimal": - row.setField(index, BigDecimal.valueOf(Long.valueOf(value))); - break; - case "java.sql.Timestamp": - row.setField(index, Timestamp.valueOf(value)); - break; - case "java.sql.Date": - row.setField(index, Date.valueOf(value)); - break; - default: - throw new RuntimeException("no support field type. the type: " + classType); - } - } - } diff --git a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java index 88cfcb8a2..c463febec 100644 --- a/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java +++ b/redis5/redis5-side/redis-side-core/src/main/java/com/dtstack/flink/sql/side/redis/table/RedisSideTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.side.redis.table; import com.dtstack.flink.sql.side.SideTableInfo; -import com.google.common.base.Preconditions; +import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; public class RedisSideTableInfo extends SideTableInfo { diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java index bb3963edb..962054f24 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/RedisOutputFormat.java @@ -24,15 +24,12 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.types.Row; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import redis.clients.jedis.*; import java.io.Closeable; import java.io.IOException; import java.util.*; public class RedisOutputFormat extends MetricOutputFormat { - private static final Logger LOG = LoggerFactory.getLogger(RedisOutputFormat.class); private String url; @@ -68,8 +65,6 @@ public class RedisOutputFormat extends MetricOutputFormat { private GenericObjectPoolConfig poolConfig; - private static int rowLenth = 1000; - private RedisOutputFormat(){ } @Override @@ -171,17 +166,7 @@ public void writeRecord(Tuple2 record) throws IOException { for (int i = 0; i < fieldNames.length; i++) { StringBuilder key = new StringBuilder(); key.append(tableName).append(":").append(perKey).append(":").append(fieldNames[i]); - - String value = "null"; - Object field = row.getField(i); - if (field != null) { - value = field.toString(); - } - jedis.set(key.toString(), value); - } - - if (outRecords.getCount()%rowLenth == 0){ - LOG.info(record.toString()); + jedis.set(key.toString(), row.getField(i).toString()); } outRecords.inc(); } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java index 2fcd20dd8..c114b946e 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisSinkParser.java @@ -21,10 +21,7 @@ import com.dtstack.flink.sql.table.AbsTableParser; import com.dtstack.flink.sql.table.TableInfo; import com.dtstack.flink.sql.util.MathUtil; -import org.apache.commons.lang3.StringUtils; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Map; public class RedisSinkParser extends AbsTableParser { @@ -45,17 +42,6 @@ public TableInfo getTableInfo(String tableName, String fieldsInfo, Map primaryKeysList = null; - if (!StringUtils.isEmpty(primaryKeysStr)) { - String[] primaryKeysArray = primaryKeysStr.split(","); - primaryKeysList = new ArrayList(Arrays.asList(primaryKeysArray)); - } else { - primaryKeysList = new ArrayList<>(); - } - redisTableInfo.setPrimaryKeys(primaryKeysList); - return redisTableInfo; } } diff --git a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java index 82def4115..7afb51cdd 100644 --- a/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java +++ b/redis5/redis5-sink/src/main/java/com/dtstack/flink/sql/sink/redis/table/RedisTableInfo.java @@ -19,7 +19,7 @@ package com.dtstack.flink.sql.sink.redis.table; import com.dtstack.flink.sql.table.TargetTableInfo; -import com.google.common.base.Preconditions; +import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions; public class RedisTableInfo extends TargetTableInfo { diff --git a/serversocket/pom.xml b/serversocket/pom.xml deleted file mode 100644 index dd76df619..000000000 --- a/serversocket/pom.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - flink.sql - com.dtstack.flink - 1.0-SNAPSHOT - ../pom.xml - - 4.0.0 - - sql.serversocket - 1.0-SNAPSHOT - - serversocket-source - - pom - - - 3.8.1 - 1.0-SNAPSHOT - - - - - junit - junit - ${junit.version} - test - - - - com.dtstack.flink - sql.core - ${sql.core.version} - provided - - - \ No newline at end of file diff --git a/serversocket/serversocket-source/pom.xml b/serversocket/serversocket-source/pom.xml deleted file mode 100644 index 9a1f274c0..000000000 --- a/serversocket/serversocket-source/pom.xml +++ /dev/null @@ -1,83 +0,0 @@ - - - - sql.serversocket - com.dtstack.flink - 1.0-SNAPSHOT - ../pom.xml - - 4.0.0 - - sql.source.serversocket - 1.0-SNAPSHOT - jar - - serversocket-source - http://maven.apache.org - - - - - org.apache.maven.plugins - maven-shade-plugin - 1.4 - - - package - - shade - - - - - org.slf4j - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - maven-antrun-plugin - 1.2 - - - copy-resources - - package - - run - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java deleted file mode 100644 index b76e94159..000000000 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/CustomerSocketTextStreamFunction.java +++ /dev/null @@ -1,183 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.flink.sql.source.serversocket; - -import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.streaming.api.functions.source.SourceFunction; -import org.apache.flink.types.Row; -import org.apache.flink.util.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.util.Iterator; - - -/** - * Reason: - * Date: 2018/12/18 - * Company: www.dtstack.com - * - * @author maqi - */ -public class CustomerSocketTextStreamFunction implements SourceFunction { - private static final Logger LOG = LoggerFactory.getLogger(CustomerSocketTextStreamFunction.class); - - /** - * Default delay between successive connection attempts. - */ - private static final int DEFAULT_CONNECTION_RETRY_SLEEP = 2000; - - /** - * Default connection timeout when connecting to the server socket (infinite). - */ - private static final int CONNECTION_TIMEOUT_TIME = 0; - - private final ObjectMapper objectMapper = new ObjectMapper(); - - /** - * Type information describing the result type. - */ - private final TypeInformation typeInfo; - - /** - * Field names to parse. Indices match fieldTypes indices. - */ - private final String[] fieldNames; - - /** - * Types to parse fields as. Indices match fieldNames indices. - */ - private final TypeInformation[] fieldTypes; - - private volatile boolean isRunning = true; - - private transient Socket currentSocket; - - ServersocketSourceTableInfo tableInfo; - - public CustomerSocketTextStreamFunction(ServersocketSourceTableInfo tableInfo, TypeInformation typeInfo) { - this.typeInfo = typeInfo; - - this.fieldNames = ((RowTypeInfo) typeInfo).getFieldNames(); - - this.fieldTypes = ((RowTypeInfo) typeInfo).getFieldTypes(); - - this.tableInfo = tableInfo; - } - - @Override - public void run(SourceContext ctx) throws Exception { - final StringBuilder buffer = new StringBuilder(); - long attempt = 0; - - while (isRunning) { - try { - Socket socket = new Socket(); - currentSocket = socket; - socket.connect(new InetSocketAddress(tableInfo.getHostname(), tableInfo.getPort()), CONNECTION_TIMEOUT_TIME); - - BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream())); - char[] cbuf = new char[8192]; - int bytesRead; - while (isRunning && (bytesRead = reader.read(cbuf)) != -1) { - buffer.append(cbuf, 0, bytesRead); - int delimPos; - String delimiter = tableInfo.getDelimiter(); - while (buffer.length() >= delimiter.length() && (delimPos = buffer.indexOf(delimiter)) != -1) { - String record = buffer.substring(0, delimPos); - // truncate trailing carriage return - if (delimiter.equals("\n") && record.endsWith("\r")) { - record = record.substring(0, record.length() - 1); - } - ctx.collect(convertToRow(record)); - buffer.delete(0, delimPos + delimiter.length()); - } - } - } catch (Exception e) { - LOG.info("Connection server failed, Please check configuration !!!!!!!!!!!!!!!!"); - } - - - // if we dropped out of this loop due to an EOF, sleep and retry - if (isRunning) { - attempt++; - if (tableInfo.getMaxNumRetries() == -1 || attempt < tableInfo.getMaxNumRetries()) { - Thread.sleep(DEFAULT_CONNECTION_RETRY_SLEEP); - } else { - // this should probably be here, but some examples expect simple exists of the stream source - // throw new EOFException("Reached end of stream and reconnects are not enabled."); - break; - } - } - } - - // collect trailing data - if (buffer.length() > 0) { - ctx.collect(convertToRow(buffer.toString())); - } - } - - public Row convertToRow(String record) throws IOException { - JsonNode root = objectMapper.readTree(record); - Row row = new Row(fieldNames.length); - for (int i = 0; i < fieldNames.length; i++) { - JsonNode node = getIgnoreCase(root, fieldNames[i]); - if (node == null) { - row.setField(i, null); - } else { - // Read the value as specified type - Object value = objectMapper.treeToValue(node, fieldTypes[i].getTypeClass()); - row.setField(i, value); - } - } - return row; - } - - - @Override - public void cancel() { - isRunning = false; - - // we need to close the socket as well, because the Thread.interrupt() function will - // not wake the thread in the socketStream.read() method when blocked. - Socket theSocket = this.currentSocket; - if (theSocket != null) { - IOUtils.closeSocket(theSocket); - } - } - - public JsonNode getIgnoreCase(JsonNode jsonNode, String key) { - Iterator iter = jsonNode.fieldNames(); - while (iter.hasNext()) { - String key1 = iter.next(); - if (key1.equalsIgnoreCase(key)) { - return jsonNode.get(key1); - } - } - return null; - } -} diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java deleted file mode 100644 index 3a67d2c98..000000000 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/ServersocketSource.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.flink.sql.source.serversocket; - -import com.dtstack.flink.sql.source.IStreamSourceGener; -import com.dtstack.flink.sql.source.serversocket.table.ServersocketSourceTableInfo; -import com.dtstack.flink.sql.table.SourceTableInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.RowTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStreamSource; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.SocketTextStreamFunction; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.types.Row; - -/** - * Reason: - * Date: 2018/12/18 - * Company: www.dtstack.com - * - * @author maqi - */ -public class ServersocketSource implements IStreamSourceGener

{ - @Override - public Table genStreamSource(SourceTableInfo sourceTableInfo, StreamExecutionEnvironment env, StreamTableEnvironment tableEnv) { - ServersocketSourceTableInfo serversocketSourceTableInfo = (ServersocketSourceTableInfo) sourceTableInfo; - - String tableName = serversocketSourceTableInfo.getName(); - - TypeInformation[] types = new TypeInformation[serversocketSourceTableInfo.getFields().length]; - for (int i = 0; i < serversocketSourceTableInfo.getFieldClasses().length; i++) { - types[i] = TypeInformation.of(serversocketSourceTableInfo.getFieldClasses()[i]); - } - - TypeInformation typeInformation = new RowTypeInfo(types, serversocketSourceTableInfo.getFields()); - - String fields = StringUtils.join(serversocketSourceTableInfo.getFields(), ","); - - CustomerSocketTextStreamFunction customerSocketTextStreamFunction = new CustomerSocketTextStreamFunction(serversocketSourceTableInfo, typeInformation); - - DataStreamSource serversocketSource = env.addSource(customerSocketTextStreamFunction, tableName, typeInformation); - - return tableEnv.fromDataStream(serversocketSource, fields); - } -} diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java deleted file mode 100644 index b52e38e59..000000000 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceParser.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.flink.sql.source.serversocket.table; - -import com.dtstack.flink.sql.table.AbsSourceParser; -import com.dtstack.flink.sql.table.TableInfo; -import com.dtstack.flink.sql.util.MathUtil; - -import java.util.Map; - -/** - * Reason: - * Date: 2018/12/18 - * Company: www.dtstack.com - * - * @author maqi - */ -public class ServersocketSourceParser extends AbsSourceParser { - @Override - public TableInfo getTableInfo(String tableName, String fieldsInfo, Map props) { - ServersocketSourceTableInfo serversocketSourceTableInfo = new ServersocketSourceTableInfo(); - serversocketSourceTableInfo.setName(tableName); - parseFieldsInfo(fieldsInfo, serversocketSourceTableInfo); - - serversocketSourceTableInfo.setHostname(MathUtil.getString(props.get(ServersocketSourceTableInfo.HOSTNAME_KEY.toLowerCase()))); - serversocketSourceTableInfo.setPort(MathUtil.getIntegerVal(props.get(ServersocketSourceTableInfo.PORT_KEY.toLowerCase()))); - serversocketSourceTableInfo.setDelimiter(MathUtil.getString(props.get(ServersocketSourceTableInfo.DELIMITER_KEY.toLowerCase()))); - serversocketSourceTableInfo.setMaxNumRetries(MathUtil.getLongVal(props.get(ServersocketSourceTableInfo.MAXNUMRETRIES_KEY.toLowerCase()))); - - serversocketSourceTableInfo.check(); - - return serversocketSourceTableInfo; - } -} diff --git a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java b/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java deleted file mode 100644 index 796728eb3..000000000 --- a/serversocket/serversocket-source/src/main/java/com/dtstack/flink/sql/source/serversocket/table/ServersocketSourceTableInfo.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.dtstack.flink.sql.source.serversocket.table; - -import com.dtstack.flink.sql.table.SourceTableInfo; -import com.google.common.base.Preconditions; - -/** - * Reason: - * Date: 2018/12/18 - * Company: www.dtstack.com - * - * @author maqi - */ -public class ServersocketSourceTableInfo extends SourceTableInfo { - //version - private static final String CURR_TYPE = "serversocket"; - - public static final String HOSTNAME_KEY = "host"; - - public static final String PORT_KEY = "port"; - - public static final String DELIMITER_KEY = "delimiter"; - - public static final String MAXNUMRETRIES_KEY = "maxNumRetries"; - - - public ServersocketSourceTableInfo() { - super.setType(CURR_TYPE); - } - - private String hostname; - - private int port; - - private String delimiter; - - private long maxNumRetries; - - public String getHostname() { - return hostname; - } - - public void setHostname(String hostname) { - this.hostname = hostname; - } - - public int getPort() { - return port; - } - - public void setPort(int port) { - this.port = port; - } - - public String getDelimiter() { - return delimiter; - } - - public void setDelimiter(String delimiter) { - this.delimiter = delimiter; - } - - public long getMaxNumRetries() { - return maxNumRetries; - } - - public void setMaxNumRetries(long maxNumRetries) { - this.maxNumRetries = maxNumRetries; - } - - - @Override - public boolean check() { - Preconditions.checkNotNull(hostname,"host name not null"); - Preconditions.checkNotNull(port,"port not null"); - Preconditions.checkNotNull(delimiter,"delimiter name not null"); - Preconditions.checkNotNull(maxNumRetries,"maxNumRetries name not null"); - - Preconditions.checkArgument(port > 0 && port < 65536, "port is out of range"); - Preconditions.checkArgument(maxNumRetries >= -1, "maxNumRetries must be zero or larger (num retries), or -1 (infinite retries)"); - return false; - } - - -} diff --git a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java index 961539fde..d123c8f9c 100644 --- a/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-all-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAllReqRow.java @@ -24,9 +24,10 @@ import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow; import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.flink.api.java.typeutils.RowTypeInfo; -import com.google.common.collect.Maps; +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.sql.Connection; import java.sql.DriverManager; import java.util.List; @@ -42,7 +43,7 @@ public class SqlserverAllReqRow extends RdbAllReqRow { private static final String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; public SqlserverAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(new SqlserverAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override diff --git a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java index cb4240ca6..003d61b27 100644 --- a/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java +++ b/sqlserver/sqlserver-side/sqlserver-async-side/src/main/java/com/dtstack/flink/sql/side/sqlserver/SqlserverAsyncReqRow.java @@ -41,7 +41,7 @@ public class SqlserverAsyncReqRow extends RdbAsyncReqRow { private final static String SQLSERVER_DRIVER = "net.sourceforge.jtds.jdbc.Driver"; public SqlserverAsyncReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List outFieldInfoList, SideTableInfo sideTableInfo) { - super(new SqlserverAsyncSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo)); + super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo); } @Override @@ -53,19 +53,11 @@ public void open(Configuration parameters) throws Exception { .put("driver_class", SQLSERVER_DRIVER) .put("max_pool_size", DEFAULT_MAX_DB_CONN_POOL_SIZE) .put("user", rdbSideTableInfo.getUserName()) - .put("password", rdbSideTableInfo.getPassword()) - .put("provider_class", DT_PROVIDER_CLASS) - .put("preferred_test_query", PREFERRED_TEST_QUERY_SQL) - .put("idle_connection_test_period", DEFAULT_IDLE_CONNECTION_TEST_PEROID) - .put("test_connection_on_checkin", DEFAULT_TEST_CONNECTION_ON_CHECKIN); - - - System.setProperty("vertx.disableFileCPResolving", "true"); + .put("password", rdbSideTableInfo.getPassword()); VertxOptions vo = new VertxOptions(); vo.setEventLoopPoolSize(DEFAULT_VERTX_EVENT_LOOP_POOL_SIZE); vo.setWorkerPoolSize(DEFAULT_VERTX_WORKER_POOL_SIZE); - vo.setFileResolverCachingEnabled(false); Vertx vertx = Vertx.vertx(vo); setRdbSQLClient(JDBCClient.createNonShared(vertx, sqlserverClientConfig)); } diff --git a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java index 2341bca66..904cb0895 100644 --- a/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java +++ b/sqlserver/sqlserver-sink/src/main/java/com/dtstack/flink/sql/sink/sqlserver/SqlserverSink.java @@ -21,7 +21,6 @@ import com.dtstack.flink.sql.sink.rdb.RdbSink; import com.dtstack.flink.sql.sink.rdb.format.ExtendOutputFormat; import com.dtstack.flink.sql.sink.rdb.format.RetractJDBCOutputFormat; -import com.dtstack.flink.sql.util.DtStringUtil; import org.apache.commons.lang3.StringUtils; import java.util.*; @@ -47,7 +46,7 @@ public RetractJDBCOutputFormat getOutputFormat() { } @Override - public void buildSql(String scheam, String tableName, List fields) { + public void buildSql(String tableName, List fields) { buildInsertSql(tableName, fields); } @@ -65,26 +64,13 @@ private void buildInsertSql(String tableName, List fields) { } @Override - public String buildUpdateSql(String scheam, String tableName, List fieldNames, Map> realIndexes, List fullField) { - StringBuilder sb = new StringBuilder(); - - sb.append("MERGE INTO " + tableName + " T1 USING " + public String buildUpdateSql(String tableName, List fieldNames, Map> realIndexes, List fullField) { + return "MERGE INTO " + tableName + " T1 USING " + "(" + makeValues(fieldNames) + ") T2 ON (" - + updateKeySql(realIndexes) + ") "); - - - String updateSql = getUpdateSql(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)); - - if (StringUtils.isNotEmpty(updateSql)) { - sb.append(" WHEN MATCHED THEN UPDATE SET "); - sb.append(updateSql); - } - - sb.append(" WHEN NOT MATCHED THEN " + + updateKeySql(realIndexes) + ") WHEN MATCHED THEN UPDATE SET " + + getUpdateSql(fieldNames, fullField, "T1", "T2", keyColList(realIndexes)) + " WHEN NOT MATCHED THEN " + "INSERT (" + quoteColumns(fieldNames) + ") VALUES (" - + quoteColumns(fieldNames, "T2") + ")"); - - return sb.toString(); + + quoteColumns(fieldNames, "T2") + ");"; }