notebook/flinksql/sqlcopy/apm-crash.sql

--********************************************************************--
-- Author:         zhougongping@1253758386586182.onaliyun.com
-- Created Time:   2023-10-11 19:44:10
-- Description:    Write your description here
-- Hints:          You can use SET statements to modify the configuration
--********************************************************************--
CREATE TEMPORARY TABLE ods_crash_log (
    errno varchar,

    gmid bigint,
    appid bigint,
    app_id varchar,
    app_ver varchar,
    app_pkg varchar,
    sdk_ver varchar,
    channel varchar,

    country varchar,
    province varchar,
    city varchar,
    isp varchar,
    ip varchar,

    device_width int,
    device_height int,
    device_id varchar,
    device_lang varchar,
    device_model varchar,
    device_brand varchar,
    device_os varchar,
    device_type varchar,
    cpu_abi varchar,
    device_ram int,
    avail_ram int,
    device_hdd int,
    avail_hdd int,
    is_root SMALLINT,
    is_front SMALLINT,
    battery_temp varchar,
    cpu_usage varchar,
    use_time BIGINT,

    crash_file varchar,
    crash_type varchar,
    brief_content varchar,
    brief_title varchar,
    is_parsed SMALLINT,

    net_type varchar,
    user_id varchar,
    server_id varchar,

    report_time TIMESTAMP
) WITH (
    'connector' = 'hologres',
    'endpoint' = 'hgprecn-cn-i7m2ssubq004-cn-hangzhou-vpc.hologres.aliyuncs.com:80',
    'username' = '${secret_values.ak}',
    'password' = '${secret_values.sk}',
    'dbname' = 'sdk_statis',
    'tablename' = 'ods_apm_crash',
    'mutatetype' = 'insertorignore',
    'ignoredelete' = 'true'
    -- 'connector'='print',
    -- 'logger'='true'
);

CREATE TEMPORARY TABLE sls_apm_log (
    app_package VARCHAR,
    app_ver VARCHAR,
    appid VARCHAR,
    channel VARCHAR,
    device VARCHAR,
    event_content VARCHAR,
    event_type VARCHAR,
    is_front SMALLINT,
    sdk_ver VARCHAR,
    server_id VARCHAR,
    use_time INT,
    user_id VARCHAR,
    __tag__ MAP<VARCHAR, VARCHAR> METADATA VIRTUAL,
    `__tag__:__X19pcF9f__` VARCHAR,
    xid AS HASHIDS_DECODE(appid, '25db81fde65423fb8470b62c5cd5766b', 12),
    device_model AS JSON_VALUE(device, '$.model')
) WITH (
    'connector' = 'sls',
    'endPoint' = 'cn-hangzhou-intranet.log.aliyuncs.com',
    'accessId' = '${secret_values.ak}',
    'accessKey' = '${secret_values.sk}',
    'startTime' = '2023-10-18 00:00:00',
    'project' = 'sh996sdk',
    'logStore' = 'apm-log',
    'consumerGroup' = 'flink-crash-sync'
);

INSERT INTO
    ods_crash_log
SELECT
    MD5(CONCAT_WS('$', crash_type, brief_content, brief_title)) AS errno,

    CAST(gmid AS BIGINT) AS gmid,
    CAST(appid AS BIGINT) AS appid,
    app_id,
    app_ver,
    app_pkg,
    sdk_ver,
    channel,

    country,
    province,
    city,
    isp,
    ip,

    CAST(CASE device_width WHEN '' THEN 0 ELSE device_width END AS INT) AS device_width,
    CAST(CASE device_height WHEN '' THEN 0 ELSE device_height END AS INT) AS device_height,
    device_id,
    device_lang,
    device_model,
    device_brand,
    device_os,
    device_type,
    cpu_abi,
    CAST(CASE device_ram WHEN '' THEN 0 ELSE device_ram END AS INT) AS device_ram,
    CAST(CASE avail_ram WHEN '' THEN 0 ELSE avail_ram END AS INT) AS avail_ram,
    CAST(CASE device_hdd WHEN '' THEN 0 ELSE device_hdd END AS INT) AS device_hdd,
    CAST(CASE avail_hdd WHEN '' THEN 0 ELSE avail_hdd END AS INT) AS avail_hdd,
    CAST(is_root AS SMALLINT) AS is_root,
    is_front,
    battery_temp,
    cpu_usage,

    use_time,

    crash_file,
    crash_type,
    brief_content,
    brief_title,
    CAST(0 AS SMALLINT) AS is_parsed,
    net_type,

    user_id,
    server_id,
    report_time

FROM (
    SELECT
        JSON_VALUE(xid, '$[0]') AS gmid,
        JSON_VALUE(xid, '$[1]') AS appid,
        appid AS app_id,
        app_ver,
        app_package AS app_pkg,
        sdk_ver,
        channel,

        '' AS country,
        '' AS province,
        '' AS city,
        '' AS isp,
        COALESCE(`__tag__:__X19pcF9f__`, __tag__['__client_ip__'], '0.0.0.0') AS ip,

        JSON_VALUE(device, '$.width') AS device_width,
        JSON_VALUE(device, '$.height') AS device_height,
        JSON_VALUE(device, '$.id') AS device_id,
        JSON_VALUE(device, '$.lang') AS device_lang,
        device_model,
        (CASE
            WHEN INSTR(LOWER(device_model), 'apple') > 0 THEN 'apple'
            WHEN INSTR(LOWER(device_model), 'iphone') > 0 THEN 'apple'
            WHEN INSTR(LOWER(device_model), 'ipad') > 0 THEN 'apple'
            ELSE SPLIT_INDEX(LOWER(device_model), ':', 0)
        END) AS device_brand,
        JSON_VALUE(device, '$.os') AS device_os,
        JSON_VALUE(device, '$.type') AS device_type,
        JSON_VALUE(device, '$.cpu_abi') AS cpu_abi,
        JSON_VALUE(device, '$.ram') AS device_ram,
        JSON_VALUE(device, '$.avail_ram') AS avail_ram,
        JSON_VALUE(device, '$.storage') AS device_hdd,
        JSON_VALUE(device, '$.avail_storage') AS avail_hdd,
        JSON_VALUE(device, '$.root') AS is_root,
        is_front,
        JSON_VALUE(device, '$.battery_temp') AS battery_temp,
        JSON_VALUE(device, '$.cpu_usage') AS cpu_usage,

        use_time,

        JSON_VALUE(event_content, '$.log_url') AS crash_file,
        JSON_VALUE(event_content, '$.type') AS crash_type,
        JSON_VALUE(event_content, '$.brief_content') AS brief_content,
        JSON_VALUE(event_content, '$.brief_title') AS brief_title,
        JSON_VALUE(device, '$.net_type') AS net_type,

        user_id,
        server_id,
        TO_TIMESTAMP_LTZ(CAST(__tag__['__receive_time__'] AS BIGINT), 0) AS report_time
    FROM
        sls_apm_log
    WHERE
        event_type = 'crash'
        AND CARDINALITY(xid) = 2
) AS t
;