From 107c2780c428c5c658054097c3f7ea264b3e504f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=B9=E8=88=9F?= <13007110208@163.com> Date: Tue, 1 Apr 2025 11:41:15 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A41?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Hologres/hologres创建实时表.sql | 77 +++++ flinksql/6182作业信息.md | 52 +++ flinksql/sqlcopy/apm-crash.sql | 201 +++++++++++ flinksql/sqlcopy/c3rq2holo_login_log.sql | 311 +++++++++++++++++ flinksql/sqlcopy/csrq2holo_login_log.sql | 311 +++++++++++++++++ flinksql/sqlcopy/game_merge.sql | 142 ++++++++ flinksql/sqlcopy/hongbao_test.sql | 111 +++++++ flinksql/sqlcopy/rds2holo_tab_user.sql | 131 ++++++++ .../sqlcopy/sls2engine_holo_login_log.sql | 312 ++++++++++++++++++ flinksql/sqlcopy/sls2holo_sdk_login_log.sql | 81 +++++ 10 files changed, 1729 insertions(+) create mode 100644 Hologres/hologres创建实时表.sql create mode 100644 flinksql/6182作业信息.md create mode 100644 flinksql/sqlcopy/apm-crash.sql create mode 100644 flinksql/sqlcopy/c3rq2holo_login_log.sql create mode 100644 flinksql/sqlcopy/csrq2holo_login_log.sql create mode 100644 flinksql/sqlcopy/game_merge.sql create mode 100644 flinksql/sqlcopy/hongbao_test.sql create mode 100644 flinksql/sqlcopy/rds2holo_tab_user.sql create mode 100644 flinksql/sqlcopy/sls2engine_holo_login_log.sql create mode 100644 flinksql/sqlcopy/sls2holo_sdk_login_log.sql diff --git a/Hologres/hologres创建实时表.sql b/Hologres/hologres创建实时表.sql new file mode 100644 index 0000000..769f906 --- /dev/null +++ b/Hologres/hologres创建实时表.sql @@ -0,0 +1,77 @@ + +SELECT * FROM ads.date_table5; + +DROP TABLE ads.date_table; + + + + +CREATE TABLE ads.base_sales( + day TEXT NOT NULL, + hour INT, + user_id BIGINT, + ts TIMESTAMPTZ, + amount FLOAT, + pk TEXT NOT NULL PRIMARY KEY +); + +INSERT INTO ads.base_sales VALUES ('2024-08-29',1,222222,'2024-08-29 16:41:19.141528+08',5,'ddd'); +ALTER TABLE ads.base_sales SET (binlog_level = replica); +INSERT INTO ads.base_sales VALUES ('2024-08-29',2,3333,'2024-08-29 17:44:19.141528+08',100,'aaaaa'); +INSERT INTO ads.base_sales VALUES ('2024-08-29',3,3333,'2024-08-29 17:44:19.141528+08',100,'ccc'); +INSERT INTO ads.base_sales VALUES ('2024-08-29',3,3333,'2024-08-29 17:44:19.141528+08',500,'gg'); + + +SET hg_experimental_enable_hybrid_incremental_mode = true; +CREATE DYNAMIC TABLE ads.sales_incremental + WITH ( + refresh_mode='incremental', + auto_refresh_enable='true', + incremental_auto_refresh_schd_start_time = 'immediate', + incremental_auto_refresh_interval = '3 minutes') +AS + SELECT day, hour, SUM(amount), COUNT(1) + FROM ads.base_sales + GROUP BY day, hour; + +select * from ads.sales_incremental; + +CREATE DYNAMIC TABLE ads.q1_full + WITH ( + refresh_mode='full', + auto_refresh_enable='true', + full_auto_refresh_interval='1 minutes' + -- ,full_guc_hg_computing_resource='serverless', + -- full_guc_hg_experimental_serverless_computing_required_cores='32' + ) +AS + SELECT day, hour, SUM(amount), COUNT(1) + FROM ads.base_sales + GROUP BY day, hour; + +select * from ads.q1_full; + +DELETE FROM ads.base_sales where hour=1 + + +CREATE DYNAMIC TABLE dt_sales_incremental + WITH ( + refresh_mode='incremental', + auto_refresh_enable='true', + incremental_auto_refresh_schd_start_time='2024-09-15 00:00:00', + incremental_auto_refresh_interval='5 minutes', + incremental_guc_hg_computing_resource='serverless', + incremental_guc_hg_experimental_serverless_computing_required_cores='128') +AS +SELECT + sale_detail.app_id, + sale_detail.uid, + product, + SUM(sale_detail.gmv) AS sum_gmv, + sale_detail.order_time, + user_info.province, + user_info.city +FROM hm.sale_detail +INNER JOIN hm.user_info FOR SYSTEM_TIME AS OF PROCTIME() +ON sale_detail.uid =user_info.uid +GROUP BY sale_detail.app_id,sale_detail.uid,sale_detail.product,sale_detail.order_time,user_info.province,user_info.city; \ No newline at end of file diff --git a/flinksql/6182作业信息.md b/flinksql/6182作业信息.md new file mode 100644 index 0000000..436e65b --- /dev/null +++ b/flinksql/6182作业信息.md @@ -0,0 +1,52 @@ +### `sls2engine_holo_login_log` +- `sls_login_log <-996engine-log login-log sls2holo_engine_login_log`（主日志表） + - `sls2engine_holo_login_log`（引擎转换日志） + - `ods_active_role` _// 角色活跃原始数据_ + - `ods_role` _// 角色基础信息_ + - `dwd_role_level` _// 角色等级明细_ + - `ods_role_duration` _// 角色在线时长_ + - `ods_platform_login` _// 平台登录记录_ + - `dws_analyze_account` _# 账户分析宽表_ + - `dws_analyze_role` _# 角色分析宽表_ + + +### `box_cloud_user_login_record` +- `ods_box_user_access_log <-prod-box-business-log box-user-access-log active_user_01` + - `box_cloud_user_login_record` + - `box_cloud_user_login_record` _mysql_ + + +### `sls-track_log-to-holo_multi` +- `sls-track_log-to-holo_multi <-sh996sdk track-log sls-track-log-to-holo-multi` + - `dws_app_props` + - `ods_active_role` + - `ods_newly_role` + - `ods_payment_object` + - `ods_payment_order` + - `ods_newly_device` + - `ods_active_device` + - `ods_active_account` + + + +sls_login_log ->996csyq-engine-log login-log csyq2holo_engine_login_log + c3rq2holo_login_log + ods_active_account + dws_analyze_role + dws_analyze_account + ods_platform_login + ods_role_duration + dwd_role_level + ods_role + ods_active_role + +sls_login_log ->chuansan login-log c3yq2holo_engine_login_log + c3rq2holo_login_log + ods_active_role + ods_role + dwd_role_level + ods_role_duration + ods_platform_login + dws_analyze_account + dws_analyze_role + ods_active_account \ No newline at end of file diff --git a/flinksql/sqlcopy/apm-crash.sql b/flinksql/sqlcopy/apm-crash.sql new file mode 100644 index 0000000..ada6771 --- /dev/null +++ b/flinksql/sqlcopy/apm-crash.sql @@ -0,0 +1,201 @@ +--********************************************************************-- +-- Author: zhougongping@1253758386586182.onaliyun.com +-- Created Time: 2023-10-11 19:44:10 +-- Description: Write your description here +-- Hints: You can use SET statements to modify the configuration +--********************************************************************-- +CREATE TEMPORARY TABLE ods_crash_log ( + errno varchar, + + gmid bigint, + appid bigint, + app_id varchar, + app_ver varchar, + app_pkg varchar, + sdk_ver varchar, + channel varchar, + + country varchar, + province varchar, + city varchar, + isp varchar, + ip varchar, + + device_width int, + device_height int, + device_id varchar, + device_lang varchar, + device_model varchar, + device_brand varchar, + device_os varchar, + device_type varchar, + cpu_abi varchar, + device_ram int, + avail_ram int, + device_hdd int, + avail_hdd int, + is_root SMALLINT, + is_front SMALLINT, + battery_temp varchar, + cpu_usage varchar, + use_time BIGINT, + + crash_file varchar, + crash_type varchar, + brief_content varchar, + brief_title varchar, + is_parsed SMALLINT, + + net_type varchar, + user_id varchar, + server_id varchar, + + report_time TIMESTAMP +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-i7m2ssubq004-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.ak}', + 'password' = '${secret_values.sk}', + 'dbname' = 'sdk_statis', + 'tablename' = 'ods_apm_crash', + 'mutatetype' = 'insertorignore', + 'ignoredelete' = 'true' + -- 'connector'='print', + -- 'logger'='true' +); + +CREATE TEMPORARY TABLE sls_apm_log ( + app_package VARCHAR, + app_ver VARCHAR, + appid VARCHAR, + channel VARCHAR, + device VARCHAR, + event_content VARCHAR, + event_type VARCHAR, + is_front SMALLINT, + sdk_ver VARCHAR, + server_id VARCHAR, + use_time INT, + user_id VARCHAR, + __tag__ MAP METADATA VIRTUAL, + `__tag__:__X19pcF9f__` VARCHAR, + xid AS HASHIDS_DECODE(appid, '25db81fde65423fb8470b62c5cd5766b', 12), + device_model AS JSON_VALUE(device, '$.model') +) WITH ( + 'connector' = 'sls', + 'endPoint' = 'cn-hangzhou-intranet.log.aliyuncs.com', + 'accessId' = '${secret_values.ak}', + 'accessKey' = '${secret_values.sk}', + 'startTime' = '2023-10-18 00:00:00', + 'project' = 'sh996sdk', + 'logStore' = 'apm-log', + 'consumerGroup' = 'flink-crash-sync' +); + +INSERT INTO + ods_crash_log +SELECT + MD5(CONCAT_WS('$', crash_type, brief_content, brief_title)) AS errno, + + CAST(gmid AS BIGINT) AS gmid, + CAST(appid AS BIGINT) AS appid, + app_id, + app_ver, + app_pkg, + sdk_ver, + channel, + + country, + province, + city, + isp, + ip, + + CAST(CASE device_width WHEN '' THEN 0 ELSE device_width END AS INT) AS device_width, + CAST(CASE device_height WHEN '' THEN 0 ELSE device_height END AS INT) AS device_height, + device_id, + device_lang, + device_model, + device_brand, + device_os, + device_type, + cpu_abi, + CAST(CASE device_ram WHEN '' THEN 0 ELSE device_ram END AS INT) AS device_ram, + CAST(CASE avail_ram WHEN '' THEN 0 ELSE avail_ram END AS INT) AS avail_ram, + CAST(CASE device_hdd WHEN '' THEN 0 ELSE device_hdd END AS INT) AS device_hdd, + CAST(CASE avail_hdd WHEN '' THEN 0 ELSE avail_hdd END AS INT) AS avail_hdd, + CAST(is_root AS SMALLINT) AS is_root, + is_front, + battery_temp, + cpu_usage, + + use_time, + + crash_file, + crash_type, + brief_content, + brief_title, + CAST(0 AS SMALLINT) AS is_parsed, + net_type, + + user_id, + server_id, + report_time + +FROM ( + SELECT + JSON_VALUE(xid, '$[0]') AS gmid, + JSON_VALUE(xid, '$[1]') AS appid, + appid AS app_id, + app_ver, + app_package AS app_pkg, + sdk_ver, + channel, + + '' AS country, + '' AS province, + '' AS city, + '' AS isp, + COALESCE(`__tag__:__X19pcF9f__`, __tag__['__client_ip__'], '0.0.0.0') AS ip, + + JSON_VALUE(device, '$.width') AS device_width, + JSON_VALUE(device, '$.height') AS device_height, + JSON_VALUE(device, '$.id') AS device_id, + JSON_VALUE(device, '$.lang') AS device_lang, + device_model, + (CASE + WHEN INSTR(LOWER(device_model), 'apple') > 0 THEN 'apple' + WHEN INSTR(LOWER(device_model), 'iphone') > 0 THEN 'apple' + WHEN INSTR(LOWER(device_model), 'ipad') > 0 THEN 'apple' + ELSE SPLIT_INDEX(LOWER(device_model), ':', 0) + END) AS device_brand, + JSON_VALUE(device, '$.os') AS device_os, + JSON_VALUE(device, '$.type') AS device_type, + JSON_VALUE(device, '$.cpu_abi') AS cpu_abi, + JSON_VALUE(device, '$.ram') AS device_ram, + JSON_VALUE(device, '$.avail_ram') AS avail_ram, + JSON_VALUE(device, '$.storage') AS device_hdd, + JSON_VALUE(device, '$.avail_storage') AS avail_hdd, + JSON_VALUE(device, '$.root') AS is_root, + is_front, + JSON_VALUE(device, '$.battery_temp') AS battery_temp, + JSON_VALUE(device, '$.cpu_usage') AS cpu_usage, + + use_time, + + JSON_VALUE(event_content, '$.log_url') AS crash_file, + JSON_VALUE(event_content, '$.type') AS crash_type, + JSON_VALUE(event_content, '$.brief_content') AS brief_content, + JSON_VALUE(event_content, '$.brief_title') AS brief_title, + JSON_VALUE(device, '$.net_type') AS net_type, + + user_id, + server_id, + TO_TIMESTAMP_LTZ(CAST(__tag__['__receive_time__'] AS BIGINT), 0) AS report_time + FROM + sls_apm_log + WHERE + event_type = 'crash' + AND CARDINALITY(xid) = 2 +) AS t +; \ No newline at end of file diff --git a/flinksql/sqlcopy/c3rq2holo_login_log.sql b/flinksql/sqlcopy/c3rq2holo_login_log.sql new file mode 100644 index 0000000..7a94ce0 --- /dev/null +++ b/flinksql/sqlcopy/c3rq2holo_login_log.sql @@ -0,0 +1,311 @@ +--********************************************************************-- +-- Author: zhougongping +-- Created Time: 2024-01-03 10:40:38 +-- Description: 传3活跃 +--********************************************************************-- +CREATE TEMPORARY TABLE sls_login_log ( + gameid INT + ,serverid INT + ,userid VARCHAR + ,schrname VARCHAR + ,saccount VARCHAR + ,nlevel INT + ,dcreatetime BIGINT + ,msgid INT + ,nonlinetime INT + ,sipaddr VARCHAR +) WITH ( + 'connector' = 'sls', + 'endPoint' = 'cn-hangzhou-intranet.log.aliyuncs.com', + 'accessId' = '${secret_values.ak}', + 'accessKey' = '${secret_values.sk}', + 'startTime' = '2024-01-02 00:00:00', + 'project' = 'chuansan', + 'logStore' = 'login-log', + 'consumerGroup' = 'c3yq2holo_engine_login_log' +); + +CREATE TEMPORARY TABLE ods_active_role ( + game_id INT + ,server_id INT + ,user_account VARCHAR + ,role_id VARCHAR + ,role_name VARCHAR + ,create_time BIGINT + ,ds VARCHAR + ,PRIMARY KEY (game_id, role_id, ds) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_active_role', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorignore' +); + +CREATE TEMPORARY TABLE ods_role ( + game_id INT + ,server_id INT + ,role_id VARCHAR + ,`level` INT + ,PRIMARY KEY (game_id, server_id, role_id) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_role', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorupdate' +); + +CREATE TEMPORARY TABLE dwd_role_level ( + game_id INT + ,server_id INT + ,role_id VARCHAR + ,ds VARCHAR + ,`level` INT + ,PRIMARY KEY (game_id, server_id, role_id, ds) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'dwd_role_level', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorupdate' +); + +CREATE TEMPORARY TABLE ods_role_duration ( + game_id INT + ,server_id INT + ,user_account VARCHAR + ,role_id VARCHAR + ,`duration` INT + ,`create_time` TIMESTAMP + ,`level` INT + ,role_name VARCHAR + + ,PRIMARY KEY (game_id, server_id, role_id, create_time) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_role_duration', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorignore' +); + +CREATE TEMPORARY TABLE ods_platform_login ( + game_id INT + ,user_id INT + ,user_account VARCHAR + ,promote_id INT + ,platform_id INT + ,login_time INT + ,login_ip VARCHAR +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_platform_login', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorignore' +); + +CREATE TEMPORARY TABLE dws_analyze_account ( + game_id BIGINT + ,server_id BIGINT + ,user_account VARCHAR + ,last_active_time BIGINT + ,PRIMARY KEY (game_id, server_id, user_account) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'dws_analyze_account', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorupdate' +); + +CREATE TEMPORARY TABLE dws_analyze_role ( + game_id BIGINT + ,server_id BIGINT + ,role_id VARCHAR + ,last_active_time BIGINT + ,PRIMARY KEY (game_id, server_id, role_id) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'dws_analyze_role', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorupdate' +); + +CREATE TEMPORARY TABLE ods_active_account ( + game_id INT + ,server_id INT + ,user_account VARCHAR + ,ds VARCHAR + ,create_time BIGINT + ,PRIMARY KEY (game_id, server_id, user_account, ds) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_active_account', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorignore' +); + +-- ============================================ +BEGIN STATEMENT SET; + +INSERT INTO + ods_active_role +SELECT + gameid AS game_id + ,serverid AS server_id + ,saccount AS user_account + ,userid AS role_id + ,schrname AS role_name + -- ,nlevel AS role_level + ,dcreatetime AS create_time + ,FROM_UNIXTIME(dcreatetime, 'yyyyMMdd') AS ds +FROM + sls_login_log; + +INSERT INTO + ods_role +SELECT + gameid AS game_id + ,serverid AS server_id + ,userid AS role_id + ,nlevel AS `level` +FROM + sls_login_log; + +INSERT INTO + dwd_role_level +SELECT + gameid AS game_id + ,serverid AS server_id + ,userid AS role_id + ,FROM_UNIXTIME(dcreatetime, 'yyyyMMdd') AS ds + ,nlevel AS `level` +FROM + sls_login_log; + +INSERT INTO + ods_role_duration +SELECT + gameid AS game_id + ,serverid AS server_id + ,saccount AS user_account + ,userid AS role_id + ,nonlinetime AS duration + ,TO_TIMESTAMP_LTZ(dcreatetime, 0) AS create_time + -- ,FROM_UNIXTIME(dcreatetime, 'yyyyMMdd') AS ds + ,nlevel AS `level` + ,schrname AS role_name +FROM + sls_login_log +WHERE + msgid = 2001 +; + +INSERT INTO + ods_platform_login +SELECT + gameid AS game_id + ,CAST(saccount AS INT) AS user_id + ,saccount AS user_account + ,0 AS promote_id + ,0 AS platform_id + ,CAST(dcreatetime AS INT) AS login_time + ,sipaddr AS login_ip +FROM + sls_login_log +WHERE + msgid = 2000 + AND REGEXP(saccount, '^\d+$') +; + +INSERT INTO + dws_analyze_account +SELECT + gameid AS game_id + ,-1 AS server_id + ,saccount AS user_account + ,CAST(dcreatetime AS BIGINT) AS last_active_time +FROM + sls_login_log +WHERE + saccount IS NOT NULL + AND msgid = 2000 +UNION ALL +SELECT + gameid AS game_id + ,serverid AS server_id + ,saccount AS user_account + ,CAST(dcreatetime AS BIGINT) AS last_active_time +FROM + sls_login_log +WHERE + saccount IS NOT NULL + AND msgid = 2000 +; + +INSERT INTO + dws_analyze_role +SELECT + gameid AS game_id + ,-1 AS server_id + ,userid AS role_id + ,CAST(dcreatetime AS BIGINT) AS last_active_time +FROM + sls_login_log +WHERE + userid IS NOT NULL + AND msgid = 2000 +UNION ALL +SELECT + gameid AS game_id + ,serverid AS server_id + ,userid AS role_id + ,CAST(dcreatetime AS BIGINT) AS last_active_time +FROM + sls_login_log +WHERE + userid IS NOT NULL + AND msgid = 2000 +; + +INSERT INTO + ods_active_account +SELECT + gameid AS game_id + ,serverid AS server_id + ,saccount AS user_account + ,FROM_UNIXTIME(dcreatetime - 1, 'yyyyMMdd') AS ds + ,(dcreatetime - 1) AS create_time +FROM + sls_login_log +; + +END; \ No newline at end of file diff --git a/flinksql/sqlcopy/csrq2holo_login_log.sql b/flinksql/sqlcopy/csrq2holo_login_log.sql new file mode 100644 index 0000000..a204c28 --- /dev/null +++ b/flinksql/sqlcopy/csrq2holo_login_log.sql @@ -0,0 +1,311 @@ +--********************************************************************-- +-- Author: zhougongping@1253758386586182.onaliyun.com +-- Created Time: 2023-02-27 19:40:02 +-- Description: Write your description here +--********************************************************************-- +CREATE TEMPORARY TABLE sls_login_log ( + gameid INT + ,serverid INT + ,userid VARCHAR + ,schrname VARCHAR + ,saccount VARCHAR + ,nlevel INT + ,dcreatetime BIGINT + ,msgid INT + ,nonlinetime INT + ,sipaddr VARCHAR +) WITH ( + 'connector' = 'sls', + 'endPoint' = 'cn-shenzhen.log.aliyuncs.com', + 'accessId' = '${secret_values.996engine_ak}', + 'accessKey' = '${secret_values.996engine_sk}', + 'startTime' = '2023-02-27 00:00:00', + 'project' = '996csyq-engine-log', + 'logStore' = 'login-log', + 'consumerGroup' = 'csyq2holo_engine_login_log' +); + +CREATE TEMPORARY TABLE ods_active_role ( + game_id INT + ,server_id INT + ,user_account VARCHAR + ,role_id VARCHAR + ,role_name VARCHAR + ,create_time BIGINT + ,ds VARCHAR + ,PRIMARY KEY (game_id, role_id, ds) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_active_role', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorignore' +); + +CREATE TEMPORARY TABLE ods_role ( + game_id INT + ,server_id INT + ,role_id VARCHAR + ,`level` INT + ,PRIMARY KEY (game_id, server_id, role_id) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_role', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorupdate' +); + +CREATE TEMPORARY TABLE dwd_role_level ( + game_id INT + ,server_id INT + ,role_id VARCHAR + ,ds VARCHAR + ,`level` INT + ,PRIMARY KEY (game_id, server_id, role_id, ds) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'dwd_role_level', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorupdate' +); + +CREATE TEMPORARY TABLE ods_role_duration ( + game_id INT + ,server_id INT + ,user_account VARCHAR + ,role_id VARCHAR + ,`duration` INT + ,`create_time` TIMESTAMP + ,`level` INT + ,role_name VARCHAR + + ,PRIMARY KEY (game_id, server_id, role_id, create_time) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_role_duration', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorignore' +); + +CREATE TEMPORARY TABLE ods_platform_login ( + game_id INT + ,user_id INT + ,user_account VARCHAR + ,promote_id INT + ,platform_id INT + ,login_time INT + ,login_ip VARCHAR +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_platform_login', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorignore' +); + +CREATE TEMPORARY TABLE dws_analyze_account ( + game_id BIGINT + ,server_id BIGINT + ,user_account VARCHAR + ,last_active_time BIGINT + ,PRIMARY KEY (game_id, server_id, user_account) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'dws_analyze_account', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorupdate' +); + +CREATE TEMPORARY TABLE dws_analyze_role ( + game_id BIGINT + ,server_id BIGINT + ,role_id VARCHAR + ,last_active_time BIGINT + ,PRIMARY KEY (game_id, server_id, role_id) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'dws_analyze_role', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorupdate' +); + +CREATE TEMPORARY TABLE ods_active_account ( + game_id INT + ,server_id INT + ,user_account VARCHAR + ,ds VARCHAR + ,create_time BIGINT + ,PRIMARY KEY (game_id, server_id, user_account, ds) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_active_account', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorignore' +); + +-- ============================================ +BEGIN STATEMENT SET; + +INSERT INTO + ods_active_role +SELECT + gameid AS game_id + ,serverid AS server_id + ,saccount AS user_account + ,userid AS role_id + ,schrname AS role_name + -- ,nlevel AS role_level + ,dcreatetime AS create_time + ,FROM_UNIXTIME(dcreatetime, 'yyyyMMdd') AS ds +FROM + sls_login_log; + +INSERT INTO + ods_role +SELECT + gameid AS game_id + ,serverid AS server_id + ,userid AS role_id + ,nlevel AS `level` +FROM + sls_login_log; + +INSERT INTO + dwd_role_level +SELECT + gameid AS game_id + ,serverid AS server_id + ,userid AS role_id + ,FROM_UNIXTIME(dcreatetime, 'yyyyMMdd') AS ds + ,nlevel AS `level` +FROM + sls_login_log; + +INSERT INTO + ods_role_duration +SELECT + gameid AS game_id + ,serverid AS server_id + ,saccount AS user_account + ,userid AS role_id + ,nonlinetime AS duration + ,TO_TIMESTAMP_LTZ(dcreatetime, 0) AS create_time + -- ,FROM_UNIXTIME(dcreatetime, 'yyyyMMdd') AS ds + ,nlevel AS `level` + ,schrname AS role_name +FROM + sls_login_log +WHERE + msgid = 2001 +; + +INSERT INTO + ods_platform_login +SELECT + gameid AS game_id + ,CAST(saccount AS INT) AS user_id + ,saccount AS user_account + ,0 AS promote_id + ,0 AS platform_id + ,CAST(dcreatetime AS INT) AS login_time + ,sipaddr AS login_ip +FROM + sls_login_log +WHERE + msgid = 2000 + AND REGEXP(saccount, '^\d+$') +; + +INSERT INTO + dws_analyze_account +SELECT + gameid AS game_id + ,-1 AS server_id + ,saccount AS user_account + ,CAST(dcreatetime AS BIGINT) AS last_active_time +FROM + sls_login_log +WHERE + saccount IS NOT NULL + AND msgid = 2000 +UNION ALL +SELECT + gameid AS game_id + ,serverid AS server_id + ,saccount AS user_account + ,CAST(dcreatetime AS BIGINT) AS last_active_time +FROM + sls_login_log +WHERE + saccount IS NOT NULL + AND msgid = 2000 +; + +INSERT INTO + dws_analyze_role +SELECT + gameid AS game_id + ,-1 AS server_id + ,userid AS role_id + ,CAST(dcreatetime AS BIGINT) AS last_active_time +FROM + sls_login_log +WHERE + userid IS NOT NULL + AND msgid = 2000 +UNION ALL +SELECT + gameid AS game_id + ,serverid AS server_id + ,userid AS role_id + ,CAST(dcreatetime AS BIGINT) AS last_active_time +FROM + sls_login_log +WHERE + userid IS NOT NULL + AND msgid = 2000 +; + +INSERT INTO + ods_active_account +SELECT + gameid AS game_id + ,serverid AS server_id + ,saccount AS user_account + ,FROM_UNIXTIME(dcreatetime - 1, 'yyyyMMdd') AS ds + ,(dcreatetime - 1) AS create_time +FROM + sls_login_log +; + +END; \ No newline at end of file diff --git a/flinksql/sqlcopy/game_merge.sql b/flinksql/sqlcopy/game_merge.sql new file mode 100644 index 0000000..63ee371 --- /dev/null +++ b/flinksql/sqlcopy/game_merge.sql @@ -0,0 +1,142 @@ +--********************************************************************-- +-- Author: zhougongping +-- Created Time: 2024-11-01 11:59:52 +-- Description: 传2、3、4合服同步 +--********************************************************************-- + +CREATE TEMPORARY TABLE cq_apply_hefu_servers +( + id int + ,startsid int + ,endsid VARCHAR + ,isconfirm int + ,merge_time int + ,game_id int + + ,PRIMARY KEY(id) NOT ENFORCED +) +WITH ( + 'connector' = 'mysql', + 'hostname' = 'rm-bp14do37zk3o0x7c1nm.mysql.rds.aliyuncs.com', + 'port' = '3306', + 'username' = 'dhsf_gm_mc', + 'password' = '${secret_values.dhsf_gm_mc_pwd}', + 'database-name' = 'dhsf_gm', + 'table-name' = 'apply_hefu_servers', + 'server-id' = '24071-24078', + 'scan.startup.mode' = 'initial', + 'server-time-zone' = 'Asia/Shanghai' +) +; + +CREATE TEMPORARY TABLE cs_apply_hefu_servers +( + id int + ,startsid int + ,endsid VARCHAR + ,isconfirm int + ,merge_time int + ,game_id int + + ,PRIMARY KEY(id) NOT ENFORCED +) +WITH ( + 'connector' = 'mysql', + 'hostname' = 'rm-wz9h3jss8diq63g3x8o.mysql.rds.aliyuncs.com', + 'port' = '3306', + 'username' = 'csyq_mc', + 'password' = '${secret_values.csyq_mc_pwd}', + 'database-name' = 'csyq_gm', + 'table-name' = 'apply_hefu_servers', + 'server-id' = '24071-24078', + 'scan.startup.mode' = 'initial', + 'server-time-zone' = 'Asia/Shanghai' +) +; + +CREATE TEMPORARY TABLE c3_apply_hefu_servers +( + id int + ,startsid int + ,endsid VARCHAR + ,isconfirm int + ,merge_time int + ,game_id int + + ,PRIMARY KEY(id) NOT ENFORCED +) +WITH ( + 'connector' = 'mysql', + 'hostname' = 'rm-bp1l808gp11p7czx4zo.mysql.rds.aliyuncs.com', + 'port' = '3306', + 'username' = 'dhsf_gm_mc', + 'password' = '${secret_values.dhsf_gm_mc_pwd}', + 'database-name' = 'c3yq_gm', + 'table-name' = 'apply_hefu_servers', + 'server-id' = '24071-24078', + 'scan.startup.mode' = 'initial', + 'server-time-zone' = 'Asia/Shanghai' +) +; + +CREATE TEMPORARY TABLE game_merge +( + src VARCHAR + ,id bigint + ,game_id int + ,start_id int + ,end_ids VARCHAR + ,isconfirm int + ,merge_time int + + ,PRIMARY KEY(src, id) NOT ENFORCED +) +WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'game_merge', + 'property-version' = '1', + 'mutatetype' = 'insertorupdate', + 'ignoredelete' = 'false' +) +; + + +INSERT INTO + game_merge +SELECT + 'cq' AS src + ,id + ,game_id + ,startsid + ,endsid + ,isconfirm + ,merge_time +FROM + cq_apply_hefu_servers +UNION ALL +SELECT + 'cs' AS src + ,id + ,game_id + ,startsid + ,endsid + ,isconfirm + ,merge_time +FROM + cs_apply_hefu_servers +UNION ALL +SELECT + 'c3' AS src + ,id + ,game_id + ,startsid + ,endsid + ,isconfirm + ,merge_time +FROM + c3_apply_hefu_servers +; \ No newline at end of file diff --git a/flinksql/sqlcopy/hongbao_test.sql b/flinksql/sqlcopy/hongbao_test.sql new file mode 100644 index 0000000..5cd3eac --- /dev/null +++ b/flinksql/sqlcopy/hongbao_test.sql @@ -0,0 +1,111 @@ +--********************************************************************-- +-- Author: machaofan@1253758386586182.onaliyun.com +-- Created Time: 2023-09-07 11:06:37 +-- Description: Write your description here +-- Hints: You can use SET statements to modify the configuration +--********************************************************************-- + +CREATE TEMPORARY TABLE ods_box_hongbao_log_pd( + d VARCHAR + ,i VARCHAR + ,t BIGINT + -- ,eventTime as TO_TIMESTAMP( CAST(t/1000 as bigint),3) + ,eventTime as TO_TIMESTAMP(t,3) + ,WATERMARK FOR eventTime AS eventTime - INTERVAL '30' SECONDS -- 定义事件时间，允许的最大窗口延迟为5s +) WITH ( + 'connector' ='sls', + 'endPoint' ='cn-shanghai.log.aliyuncs.com', + 'accessId' ='LTAI5tEqx7wh38NsaFW7BJFN', + 'accessKey' ='cENYqeXBbFkVZeShFyPgTlIt1ouMpa', + 'startTime' ='2023-09-07 00:00:00', + 'project' ='996box-dev', + 'logStore' ='collect-dev', + 'consumerGroup' ='hongbaotest02' +); + + +-- CREATE TEMPORARY TABLE ads_box_red_ad_statis_pd ( +-- red_id bigint, +-- user_id bigint, +-- red_show_num bigint, +-- red_click_num bigint, +-- red_into_users bigint, +-- gmt_create TIMESTAMP, +-- gmt_modify TIMESTAMP, +-- ds VARCHAR, +-- PRIMARY KEY (red_id, user_id,ds) NOT ENFORCED +-- ) WITH ( +-- -- 'connector' = 'print', +-- -- 'logger'='true' +-- 'connector'='hologres', +-- 'dbname'='sh996box_dev', +-- 'tablename'='ads_box_red_ad_statis_pd', +-- 'endpoint' = 'hgpostcn-cn-7mz2qe6e7002-cn-hangzhou-vpc.hologres.aliyuncs.com:80', +-- 'username' = '${secret_values.996engine_ak}', +-- 'password' = '${secret_values.996engine_sk}', +-- 'mutatetype'='insertorignore' +-- ); + + +CREATE TEMPORARY TABLE ads_box_test ( + red_id bigint, + user_id bigint, + red_show_num bigint, + red_click_num bigint, + red_into_users bigint, + gmt_create TIMESTAMP, + gmt_modify TIMESTAMP, + ds VARCHAR, +PRIMARY KEY (red_id, user_id,ds) NOT ENFORCED +) WITH ( +-- 'connector' = 'print', +-- 'logger'='true' + 'connector'='hologres', + 'dbname'='sh996box_dev', + 'tablename'='ads_box_test', + 'endpoint' = 'hgpostcn-cn-7mz2qe6e7002-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'mutatetype'='insertorignore' +); + + +create TEMPORARY view ads_box_red_ad_statis_pd1 as +select + i + ,eventTime + ,cast(JSON_VALUE(d,'$.uid') as BIGINT) as uid + ,cast(JSON_VALUE(d,'$.object_id') AS BIGINT) as send_id + ,cast(JSON_VALUE(d,'$.red_id') AS BIGINT) as red_id +from ods_box_hongbao_log_pd where i in ('hongbao_1300','hongbao_1301','hongbao_1302'); + +-- -- 每分钟更新一次从零点开始截止到当前时刻的累计销售额 +INSERT INTO ads_box_red_ad_statis_pd +SELECT + red_id, + send_id, + COALESCE(max(case WHEN i='hongbao_1300' THEN COALESCE(red_show_num,0) end ),0) as red_show_num, + COALESCE(max(case WHEN i='hongbao_1301' THEN COALESCE(red_click_num,0) end ),0) as red_click_num, + COALESCE(max(case WHEN i='hongbao_1302' THEN COALESCE(red_click_num,0) end ),0) as red_into_users, + window_start, + window_end , + SUBSTR(window_start,1,19) as ds +FROM +(SELECT + window_start, + window_end, + send_id, + i, + red_id, + count(uid) as red_show_num, + count(distinct uid) as red_click_num -- 当天累计销售额 +FROM TABLE( + -- 定义窗口最大长度为一天的累计窗口，窗口滚动步长为1分钟 + CUMULATE( + TABLE ads_box_red_ad_statis_pd1, + DESCRIPTOR(eventTime), + INTERVAL '1' MINUTES, + INTERVAL '1' DAY)) +GROUP BY window_start, window_end,send_id,i,red_id) as a1 +GROUP BY SUBSTR(window_start,1,19) ,window_start, window_end,send_id,red_id +; \ No newline at end of file diff --git a/flinksql/sqlcopy/rds2holo_tab_user.sql b/flinksql/sqlcopy/rds2holo_tab_user.sql new file mode 100644 index 0000000..1ff30a8 --- /dev/null +++ b/flinksql/sqlcopy/rds2holo_tab_user.sql @@ -0,0 +1,131 @@ +--********************************************************************-- +-- Author: zhougongping@1253758386586182.onaliyun.com +-- Created Time: 2023-03-22 16:51:18 +-- Description: Write your description here +--********************************************************************-- +CREATE TEMPORARY TABLE tab_user ( + id BIGINT + ,account STRING + ,promote_id INT + ,phone STRING + ,idcard STRING + ,register_way TINYINT + ,register_type TINYINT + ,register_time BIGINT + ,login_time BIGINT + ,register_ip STRING + ,login_ip STRING + ,is_union TINYINT + ,age_status TINYINT + ,is_wechat_subscribe TINYINT + ,platform_id INT + ,box_account_time BIGINT + ,channel STRING + ,idcard_time BIGINT + + ,PRIMARY KEY(id) NOT ENFORCED +) WITH ( + 'connector' = 'mysql', + 'hostname' = 'rm-bp1uj139ua9dck8n4jm.mysql.rds.aliyuncs.com', + 'port' = '3306', + 'username' = '${secret_values.rds_engine_user}', + 'password' = '${secret_values.rds_engine_pass}', + 'database-name' = 'platform', + 'table-name' = 'tab_user', + 'server-id' = '24061-24068', + 'scan.startup.mode' = 'initial', + 'server-time-zone' = 'Asia/Shanghai' +); + +CREATE TEMPORARY TABLE ods_platform_account ( + uid bigint + ,account STRING + ,promote_id bigint + ,phone STRING + ,idcard STRING + ,birthday STRING -- YYYYmmdd + ,gender SMALLINT -- 1男2女0未知 + ,reg_way SMALLINT -- 注册方式 0游客1账号 2 手机 3微信 4QQ 5百度 6微博 7 sdk8后台添加主播 + ,reg_type SMALLINT -- 注册来源 0pc 2app + ,reg_time TIMESTAMP + ,reg_ip STRING + ,is_union SMALLINT -- 是否联盟站下的用户 0否/1是 + ,age_status SMALLINT -- 0 未认证 2审核通过成年 3审核通过未成年 + ,is_wechat_subscribe SMALLINT -- 是否关注微信公众号(0否/1是) + ,platform_id INT + ,box_account_time BIGINT -- 更新盒子号时间 + ,box_channel STRING -- 盒子注册渠道 -1是游戏注册 0是官方 其余是各渠道 + ,auth_time TIMESTAMP -- 实名认证时间 + + ,PRIMARY KEY(uid) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_platform_account', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorreplace' +); + +CREATE TEMPORARY TABLE biz_account_profile ( + user_id varchar + ,_update_time_ TIMESTAMP + ,birthday varchar + ,gender int + ,PRIMARY KEY (user_id) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-i7m2ssubq004-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.ak}', + 'password' = '${secret_values.sk}', + 'dbname' = 'sdk_statis', + 'tablename' = 'biz.dwd_event_analyze_account', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorupdate' +); + +-- ================================= +BEGIN STATEMENT SET; + +INSERT INTO + ods_platform_account +SELECT + id as uid + ,account as account + ,promote_id + ,MD5(CONCAT('0', phone)) as phone + ,MD5(CONCAT('0', idcard)) as idcard + ,IF(CHAR_LENGTH(idcard) = 15, CONCAT('19', SUBSTRING(idcard, 7, 6)), SUBSTRING(idcard, 7, 8)) as birthday + ,CAST(IF(CHAR_LENGTH(idcard) = 15, SUBSTRING(idcard, 15, 1), SUBSTRING(idcard, 17, 1)) AS SMALLINT) as gender + ,register_way as reg_way + ,register_type as reg_type + ,TO_TIMESTAMP_LTZ(COALESCE(register_time, login_time, 0), 0) as reg_time + ,IFNULL(register_ip, login_ip) as reg_ip + ,is_union + ,age_status + ,is_wechat_subscribe + ,platform_id + ,box_account_time + ,channel as box_channel + ,TO_TIMESTAMP_LTZ(IF(idcard IS NULL, 0, COALESCE(idcard_time, register_time, login_time, 0)), 0) as auth_time +FROM + tab_user +; + +INSERT INTO + biz_account_profile +SELECT + CAST(id AS VARCHAR) AS user_id + ,TO_TIMESTAMP_LTZ(COALESCE(idcard_time, register_time, login_time, UNIX_TIMESTAMP()), 0) AS _update_time_ + ,IF(CHAR_LENGTH(idcard) = 15, CONCAT('19', SUBSTRING(idcard, 7, 6)), SUBSTRING(idcard, 7, 8)) AS birthday + ,CAST(IF(CHAR_LENGTH(idcard) = 15, SUBSTRING(idcard, 15, 1), SUBSTRING(idcard, 17, 1)) AS INT) AS gender +FROM + tab_user +WHERE + idcard IS NOT NULL + AND idcard <> '' +; + +END; diff --git a/flinksql/sqlcopy/sls2engine_holo_login_log.sql b/flinksql/sqlcopy/sls2engine_holo_login_log.sql new file mode 100644 index 0000000..40c477f --- /dev/null +++ b/flinksql/sqlcopy/sls2engine_holo_login_log.sql @@ -0,0 +1,312 @@ +--********************************************************************-- +-- Author: zhougongping@1253758386586182.onaliyun.com +-- Created Time: 2022-12-05 16:58:10 +-- Description: Write your description here +--********************************************************************-- +CREATE TEMPORARY TABLE sls_login_log ( + gameid INT + ,serverid INT + ,userid VARCHAR + ,schrname VARCHAR + ,saccount VARCHAR + ,nlevel INT + ,dcreatetime BIGINT + ,msgid INT + ,nonlinetime INT + ,sipaddr VARCHAR +) WITH ( + 'connector' = 'sls', + 'endPoint' = 'cn-hangzhou-intranet.log.aliyuncs.com', + 'accessId' = '${secret_values.996engine_ak}', + 'accessKey' = '${secret_values.996engine_sk}', + 'startTime' = '2022-12-01 00:00:00', + 'project' = '996engine-log', + 'logStore' = 'login-log', + 'consumerGroup' = 'sls2holo_engine_login_log' +); + +CREATE TEMPORARY TABLE ods_active_role ( + game_id INT + ,server_id INT + ,user_account VARCHAR + ,role_id VARCHAR + ,role_name VARCHAR + ,create_time BIGINT + ,ds VARCHAR + ,PRIMARY KEY (game_id, role_id, ds) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_active_role', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorignore' +); + +CREATE TEMPORARY TABLE ods_role ( + game_id INT + ,server_id INT + ,role_id VARCHAR + ,`level` INT + ,PRIMARY KEY (game_id, server_id, role_id) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_role', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorupdate' +); + +CREATE TEMPORARY TABLE dwd_role_level ( + game_id INT + ,server_id INT + ,role_id VARCHAR + ,ds VARCHAR + ,`level` INT + ,PRIMARY KEY (game_id, server_id, role_id, ds) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'dwd_role_level', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorupdate' +); + +CREATE TEMPORARY TABLE ods_role_duration ( + game_id INT + ,server_id INT + ,user_account VARCHAR + ,role_id VARCHAR + ,`duration` INT + ,`create_time` TIMESTAMP + ,`level` INT + ,role_name VARCHAR + + ,PRIMARY KEY (game_id, server_id, role_id, create_time) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_role_duration', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorignore' +); + +CREATE TEMPORARY TABLE ods_platform_login ( + game_id INT + ,user_id INT + ,user_account VARCHAR + ,promote_id INT + ,platform_id INT + ,login_time INT + ,login_ip VARCHAR +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_platform_login', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorignore' +); + +CREATE TEMPORARY TABLE dws_analyze_account ( + game_id BIGINT + ,server_id BIGINT + ,user_account VARCHAR + ,last_active_time BIGINT + ,PRIMARY KEY (game_id, server_id, user_account) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'dws_analyze_account', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorupdate' +); + +CREATE TEMPORARY TABLE dws_analyze_role ( + game_id BIGINT + ,server_id BIGINT + ,role_id VARCHAR + ,last_active_time BIGINT + ,PRIMARY KEY (game_id, server_id, role_id) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'dws_analyze_role', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorupdate' +); + +CREATE TEMPORARY TABLE ods_active_account ( + game_id INT + ,server_id INT + ,user_account VARCHAR + ,ds VARCHAR + ,create_time BIGINT + ,PRIMARY KEY (game_id, server_id, user_account, ds) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_active_account', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorignore' +); + +-- ============================================ +BEGIN STATEMENT SET; + +INSERT INTO + ods_active_role +SELECT + gameid AS game_id + ,serverid AS server_id + ,saccount AS user_account + ,userid AS role_id + ,schrname AS role_name + -- ,nlevel AS role_level + ,(dcreatetime - 1) AS create_time + ,FROM_UNIXTIME(dcreatetime - 1, 'yyyyMMdd') AS ds +FROM + sls_login_log; + +INSERT INTO + ods_role +SELECT + gameid AS game_id + ,serverid AS server_id + ,userid AS role_id + ,nlevel AS `level` +FROM + sls_login_log; + +INSERT INTO + dwd_role_level +SELECT + gameid AS game_id + ,serverid AS server_id + ,userid AS role_id + ,FROM_UNIXTIME(dcreatetime, 'yyyyMMdd') AS ds + ,nlevel AS `level` +FROM + sls_login_log; + +INSERT INTO + ods_role_duration +SELECT + gameid AS game_id + ,serverid AS server_id + ,saccount AS user_account + ,userid AS role_id + ,nonlinetime AS duration + ,TO_TIMESTAMP_LTZ(dcreatetime, 0) AS create_time + -- ,FROM_UNIXTIME(dcreatetime, 'yyyyMMdd') AS ds + ,nlevel AS `level` + ,schrname AS role_name +FROM + sls_login_log +WHERE + msgid = 2001 +; + +INSERT INTO + ods_platform_login +SELECT + gameid AS game_id + ,CAST(saccount AS INT) AS user_id + ,saccount AS user_account + ,0 AS promote_id + ,0 AS platform_id + ,CAST(dcreatetime AS INT) AS login_time + ,sipaddr AS login_ip +FROM + sls_login_log +WHERE + msgid = 2000 + AND REGEXP(saccount, '^\d+$') +; + +INSERT INTO + dws_analyze_account +SELECT + gameid AS game_id + ,-1 AS server_id + ,saccount AS user_account + ,CAST(dcreatetime AS BIGINT) AS last_active_time +FROM + sls_login_log +WHERE + saccount IS NOT NULL + AND msgid = 2000 +UNION ALL +SELECT + gameid AS game_id + ,serverid AS server_id + ,saccount AS user_account + ,CAST(dcreatetime AS BIGINT) AS last_active_time +FROM + sls_login_log +WHERE + saccount IS NOT NULL + AND msgid = 2000 +; + +INSERT INTO + dws_analyze_role +SELECT + gameid AS game_id + ,-1 AS server_id + ,userid AS role_id + ,CAST(dcreatetime AS BIGINT) AS last_active_time +FROM + sls_login_log +WHERE + userid IS NOT NULL + AND msgid = 2000 +UNION ALL +SELECT + gameid AS game_id + ,serverid AS server_id + ,userid AS role_id + ,CAST(dcreatetime AS BIGINT) AS last_active_time +FROM + sls_login_log +WHERE + userid IS NOT NULL + AND msgid = 2000 +; + +-- INSERT INTO +-- ods_active_account +-- SELECT +-- gameid AS game_id +-- ,serverid AS server_id +-- ,saccount AS user_account +-- ,FROM_UNIXTIME(dcreatetime - 1, 'yyyyMMdd') AS ds +-- ,(dcreatetime - 1) AS create_time +-- FROM +-- sls_login_log +-- WHERE +-- saccount IS NOT NULL; + +END; \ No newline at end of file diff --git a/flinksql/sqlcopy/sls2holo_sdk_login_log.sql b/flinksql/sqlcopy/sls2holo_sdk_login_log.sql new file mode 100644 index 0000000..0834899 --- /dev/null +++ b/flinksql/sqlcopy/sls2holo_sdk_login_log.sql @@ -0,0 +1,81 @@ +--********************************************************************-- +-- Author: zhougongping@1253758386586182.onaliyun.com +-- Created Time: 2023-04-11 15:17:43 +-- Description: Write your description here +--********************************************************************-- +CREATE TEMPORARY TABLE sls_com_login_log ( + app_ver VARCHAR + ,appid VARCHAR + ,channel VARCHAR + ,device_height INT + ,device_id VARCHAR + ,device_lang VARCHAR + ,device_model VARCHAR + ,device_os VARCHAR + ,device_type VARCHAR + ,device_width INT + ,event_time INT + ,ip VARCHAR + ,net_type VARCHAR + ,user_id INT +) WITH ( + 'connector' = 'sls', + 'endPoint' = 'cn-hangzhou-intranet.log.aliyuncs.com', + 'accessId' = '${secret_values.996engine_ak}', + 'accessKey' = '${secret_values.996engine_sk}', + -- 'startTime' = '2023-04-11 00:00:00', + 'project' = 'game-sdk', + 'logStore' = 'com-login', + 'consumerGroup' = 'sls2holo_com_login' +); + +CREATE TEMPORARY TABLE ods_user_profile_login ( + app_ver VARCHAR + ,appid VARCHAR + ,channel VARCHAR + ,device_id VARCHAR + ,device_lang VARCHAR + ,device_model VARCHAR + ,device_os VARCHAR + ,device_type VARCHAR + ,device_height INT + ,device_width INT + ,ip VARCHAR + ,net_type VARCHAR + ,user_id INT + ,event_time TIMESTAMP + ,ds VARCHAR + + ,PRIMARY KEY (appid, device_id, ds) NOT ENFORCED +) WITH ( + 'connector' = 'hologres', + 'endpoint' = 'hgprecn-cn-n8m2xw5ez001-cn-hangzhou-vpc.hologres.aliyuncs.com:80', + 'username' = '${secret_values.996engine_ak}', + 'password' = '${secret_values.996engine_sk}', + 'dbname' = 'dhsf_engine', + 'tablename' = 'ods_user_profile_login', + 'ignoredelete' = 'true', + 'mutatetype' = 'insertorignore' +); + +INSERT INTO + ods_user_profile_login +SELECT + app_ver + ,appid + ,channel + ,device_id + ,device_lang + ,device_model + ,device_os + ,device_type + ,device_height + ,device_width + ,ip + ,net_type + ,user_id + ,TO_TIMESTAMP_TZ(FROM_UNIXTIME(event_time, 'yyyy-MM-dd HH:mm:ss'), 'Asia/Shanghai') AS event_time + ,FROM_UNIXTIME(event_time, 'yyyyMMdd') AS ds +FROM + sls_com_login_log +; \ No newline at end of file