Merge "startop: Add script to analyze block I/O from an ftrace file"

/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

-- filter for atrace writes

CREATE VIEW IF NOT EXISTS tracing_mark_writes AS

    SELECT *

      FROM raw_ftrace_entries

     WHERE function = 'tracing_mark_write';

-- split the tracing_mark_write function args by ||s

DROP TABLE IF exists tracing_mark_write_split_array;

CREATE TABLE tracing_mark_write_split_array (

    predictorset_id INT REFERENCES raw_ftrace_entries (id),

    predictor_name,

    rest,

    gen,

    UNIQUE(predictorset_id, gen) -- drops redundant inserts into table

);

CREATE INDEX "tracing_mark_write_split_array_id" ON tracing_mark_write_split_array (

    predictorset_id COLLATE BINARY COLLATE BINARY

);

INSERT INTO tracing_mark_write_split_array

  WITH 

    split(predictorset_id, predictor_name, rest, gen) AS (

      -- split by |

      SELECT id, '', function_args || '|', 0 FROM tracing_mark_writes WHERE id

       UNION ALL

      SELECT predictorset_id, 

             substr(rest, 0, instr(rest, '|')),

             substr(rest, instr(rest, '|')+1),

             gen + 1

        FROM split

       WHERE rest <> ''),

     split_results AS (

       SELECT * FROM split WHERE predictor_name <> ''

     )

  SELECT * from split_results

;

/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

-- use the 'launching: $process_name' async slice to figure out launch duration.

DROP VIEW IF EXISTS launch_durations_named;

CREATE VIEW launch_durations_named AS

WITH

    launch_traces_raw AS (

        SELECT *

        FROM tracing_mark_write_split AS tmw,

             raw_ftrace_entries AS rfe

        WHERE atrace_message LIKE 'launching: %' AND rfe.id = tmw.raw_ftrace_entry_id

    ),

    launch_traces_joined AS (

        SELECT started.timestamp AS started_timestamp,

               finished.timestamp AS finished_timestamp,

               started.id AS started_id,

               finished.id AS finished_id,

               SUBSTR(started.atrace_message, 12) AS proc_name   -- crop out "launching: " from the string.

        FROM launch_traces_raw AS started,

             launch_traces_raw AS finished

        -- async slices ('S' -> 'F') have matching counters given the same PID.

        WHERE started.atrace_type == 'S'

              AND finished.atrace_type == 'F'

              AND started.atrace_count == finished.atrace_count

              AND started.atrace_pid == finished.atrace_pid

    )

SELECT * from launch_traces_joined;

SELECT * FROM launch_durations_named;

/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

DROP VIEW IF EXISTS blocked_iowait_for_app_launches;

CREATE VIEW blocked_iowait_for_app_launches AS

WITH

    block_launch_join AS (

        SELECT *

        FROM blocking_durations AS bd,

             launch_durations_named AS ld

        WHERE bd.block_timestamp >= ld.started_timestamp

              AND bd.unblock_timestamp <= ld.finished_timestamp

    ),

    blocked_ui_threads AS (

        SELECT *

        FROM start_process_ui_threads AS sp,

             block_launch_join AS blj

        WHERE sp.atm_ui_thread_tid == unblock_pid

              AND sp.process_name = blj.proc_name

    ),

    summed_raw AS (

        SELECT SUM(unblock_timestamp-block_timestamp)*1000 AS sum_block_duration_ms,

               *

        FROM blocked_ui_threads

        GROUP BY unblock_pid

    ),

    summed_neat AS (

        SELECT sum_block_duration_ms AS blocked_iowait_duration_ms,

               process_name,

               (finished_timestamp - started_timestamp) * 1000 AS launching_duration_ms,

               started_timestamp * 1000 AS launching_started_timestamp_ms,

               finished_timestamp * 1000 AS launching_finished_timestamp_ms

                -- filter out the rest because its just selecting 1 arbitrary row (due to the SUM aggregate).,

        FROM summed_raw

    )

SELECT * FROM summed_neat;

SELECT * FROM blocked_iowait_for_app_launches;

/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

DROP VIEW IF EXISTS sched_switch_iowaits_pre;

-- scan for the closest pair such that:

--                  sched_block_reason pid=$PID iowait=1 ...

--                  ...

--                  sched_switch next_pid=$PID

CREATE VIEW sched_switch_iowaits_pre AS

    SELECT MAX(sbr.id) AS blocked_id,

           ss.id AS sched_switch_id,

           pid,         -- iow.pid

           iowait,      -- iowait=0 or iowait=1

           caller,

           sbr_f.timestamp AS blocked_timestamp,

           ss_f.timestamp AS sched_switch_timestamp,

           next_comm,   -- name of next_pid

           next_pid     -- same as iow.pid

    FROM sched_blocked_reasons AS sbr,

         raw_ftrace_entries AS sbr_f,

         sched_switches AS ss,

         raw_ftrace_entries AS ss_f

    WHERE sbr_f.id == sbr.id

          AND ss_f.id == ss.id

          AND sbr.pid == ss.next_pid

          AND sbr.iowait = 1

          AND sbr_f.timestamp < ss_f.timestamp     -- ensures the 'closest' sched_blocked_reason is selected.

    GROUP BY ss.id

;

DROP VIEW IF EXISTS sched_switch_iowaits;

CREATE VIEW sched_switch_iowaits AS

    SELECT *, MIN(sched_switch_timestamp) AS ss_timestamp      -- drop all of the 'too large' sched_switch entries except the closest one.

    FROM sched_switch_iowaits_pre

    GROUP BY blocked_id;

SELECT * FROM sched_switch_iowaits;

-- use a real table here instead of a view, otherwise SQLiteStudio segfaults for some reason.

DROP TABLE IF EXISTS blocking_durations;

CREATE TABLE blocking_durations AS

WITH

    blocking_durations_raw AS (

        SELECT MAX(ss.id) AS block_id,

               ssf.timestamp AS block_timestamp,

               iow.sched_switch_timestamp AS unblock_timestamp,

               ss.prev_comm as block_prev_comm,

               iow.next_comm AS unblock_next_comm,

               ss.prev_state AS block_prev_state,

               iow.sched_switch_id AS unblock_id,

               iow.pid AS unblock_pid,

               iow.caller AS unblock_caller

        FROM sched_switches AS ss,          -- this is the sched_switch that caused a block (in the future when it unblocks, the reason is iowait=1).

             sched_switch_iowaits AS iow,    -- this is the sched_switch that removes the block (it is now running again).

             raw_ftrace_entries AS ssf

        WHERE ssf.id = ss.id AND ss.prev_pid == iow.next_pid AND ssf.timestamp < iow.sched_switch_timestamp

        GROUP BY unblock_timestamp 

    ),

    blocking_durations_tmp AS (

        SELECT block_id,

               unblock_timestamp,

               block_timestamp,

               block_prev_comm as comm,

               block_prev_state as block_state,

               unblock_id,

               unblock_pid,

               unblock_caller

        FROM blocking_durations_raw

    )

    SELECT * FROM blocking_durations_tmp;-- ORDER BY block_id ASC;

    --SELECT SUM(block_duration_ms) AS sum, * FROM blocking_durations GROUP BY unblock_pid ORDER BY sum DESC;

DROP INDEX IF EXISTS "blocking_durations_block_timestamp";

CREATE INDEX "blocking_durations_block_timestamp" ON blocking_durations (

    block_timestamp COLLATE BINARY COLLATE BINARY

);

DROP INDEX IF EXISTS "blocking_durations_unblock_timestamp";

CREATE INDEX "blocking_durations_unblock_timestamp" ON blocking_durations (

    unblock_timestamp COLLATE BINARY COLLATE BINARY

);

SELECT * FROM blocking_durations;

/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

DROP VIEW IF EXISTS sched_switch_next_comm_pids;

CREATE VIEW IF NOT EXISTS sched_switch_next_comm_pids AS

-- TODO: switch to using sched_switches table.

WITH

    sched_switchs AS (

        SELECT * FROM raw_ftrace_entries WHERE function = 'sched_switch' AND function_args LIKE '% next_pid=%' AND function_args NOT LIKE '% next_comm=main %'

    ),

    comm_and_pids_raws AS (

        SELECT id,

               SUBSTR(function_args, instr(function_args, "next_comm="), instr(function_args, "next_pid=") - instr(function_args, "next_comm=")) AS next_comm_raw,

               SUBSTR(function_args, instr(function_args, "next_pid="), instr(function_args, "next_prio=") - instr(function_args, "next_pid=")) AS next_pid_raw

        FROM sched_switchs

    ),

    comm_and_pids AS (

        SELECT id,

               id AS raw_ftrace_entry_id,

               TRIM(SUBSTR(next_pid_raw, 10)) AS next_pid, -- len("next_pid=") is 10

               TRIM(SUBSTR(next_comm_raw, 11)) AS next_comm -- len("next_comm=") is 11

        FROM comm_and_pids_raws

    )

SELECT * from comm_and_pids;

SELECT * from sched_switch_next_comm_pids;