flink（十四）：sql版連接器和窗口實例

說明

• 本博客週五更新一次
• Flink Sql 支持衆多連接器，語句各有不同，使用時查找起來麻煩，找到了也可能是錯的，因此我整理收集了已知的連接與窗口實例，並持續更新，在此分享出來。

分享

• 大數據博客列表

資料

• datagen文檔
• jdbc
• kafka
• hive
  • 基礎文檔 不同版本hive使用不同jar
  • hive讀寫文檔
• hbase文檔

實例

連接器

隨機數連接器

• 生成隨機數表 datagen ，每5秒生成一次

CREATE TABLE source_table (
  f0 INT,
  f1 INT,
  f2 STRING
 ) WITH (
  'connector' = 'datagen',
  'rows-per-second'='5'
 );

輸出連接器

CREATE TABLE print_table (
  f0 INT,
  f1 INT,
  f2 STRING
) WITH (
  'connector' = 'print'
);

kafka連接器

CREATE TABLE input_kafka (
  `user_id` BIGINT,
  `page_id` BIGINT,
  `status` STRING 
) WITH (
  'connector' = 'kafka',
  'topic' = 'input_kafka',
  'properties.bootstrap.servers' = 'localhost:9092',
  'properties.group.id' = 'testGroup',
  'scan.startup.mode' = 'latest-offset',
  'format' = 'json',
  --以下非必填
  'json.fail-on-missing-field' = 'false',
  'json.ignore-parse-errors' = 'true',
  'properties.zookeeper.connect' = '172.25.20.76:2181/kafka'
)

hive連接器

• hive表 必須要開啓checkpoint

CREATE CATALOG testmyhive WITH (
    'type' = 'hive',
    'default-database' = 'zhp',
    'hive-conf-dir' = '/Users/huipeizhu/hive-conf'
);

USE CATALOG testmyhive;

--- 清空表，防止多餘
drop table IF EXISTS item_test;

drop table IF EXISTS hive_flink_table;

---- 創建kafka表
create table item_test ( 
     itemId BIGINT,
     price BIGINT,
     proctime AS PROCTIME ()
 )with ( 
     'connector' = 'kafka',
     'topic' = 'flink-catalog-v1',  
     'properties.bootstrap.servers'='127.0.0.1:9092',
     'properties.group.id'='test-1',
     'format'='json',
     'scan.startup.mode' = 'earliest-offset'
 );

--- 創建hive表
SET table.sql-dialect=hive;

CREATE TABLE hive_flink_table (
 itemId BIGINT, 
 price BIGINT, 
 ups string
) TBLPROPERTIES (
  'sink.rolling-policy.rollover-interval'='1min',
  'sink.partition-commit.trigger'='process-time',
  'sink.partition-commit.policy.kind'='metastore,success-file'
);

SET table.sql-dialect=default;

insert into hive_flink_table select itemId,price, 'XXXXaaa' as ups from item_test;

mysql連接器

CREATE TABLE sync_test_1 (
  day_time string,
  total_gmv bigint,
  PRIMARY KEY (day_time) NOT ENFORCED
 ) WITH (
   'connector' = 'jdbc',
   'url' = 'jdbc:mysql://172.25.21.10:3306/flink_web?characterEncoding=UTF-8',
   'table-name' = 'sync_test_1',
   'username' = 'videoweb',
   'password' = 'suntek'
 );

Elasticsearch連接器

CREATE TABLE enriched_orders (
  order_id INT,
  order_date TIMESTAMP(0),
  customer_name STRING,
  price DECIMAL(10, 5),
  product_id INT,
  order_status BOOLEAN,
  product_name STRING,
  product_description STRING,
  shipment_id INT,
  origin STRING,
  destination STRING,
  is_arrived BOOLEAN,
  PRIMARY KEY (order_id) NOT ENFORCED
) WITH (
    'connector' = 'elasticsearch-7',
    'hosts' = 'http://172.25.23.15:9401',
    'index' = 'enriched_orders'
);

hbase

CREATE TABLE hTable (
 rowkey INT,
 family1 ROW<q1 INT>,
 family2 ROW<q2 STRING, q3 BIGINT>,
 family3 ROW<q4 DOUBLE, q5 BOOLEAN, q6 STRING>,
 PRIMARY KEY (rowkey) NOT ENFORCED
) WITH (
  --- hbase 1.4.x 對應1.4   2.2.x 對應 2.2
 'connector' = 'hbase-1.4',
 'table-name' = 'mytable',
 'zookeeper.quorum' = 'localhost:2181'
);

窗口

滾動窗口

• 滾動窗口表，其中 createTime 爲自定義時間戳字段，每5秒執行一次。

select 
  userId,count(*) as orderCount,max(money) as maxMoney,min(money) as minMoney 
  from t_order 
group by 
  userId,
  tumble(createTime,Interval '5' SECOND)

總結

• 好好學習，好好總結，努力成長。
• 同一個知識不同時間理解的廣度和深度有很大不同，一段時間後，面對同一事物，如果與上次無異，只能說明自己在原地踏步。