Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add fine-grained timings to Writers #555

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Add fine-grained timings to Writers #555

wants to merge 2 commits into from

Conversation

knighton
Copy link
Contributor

@knighton knighton commented Dec 30, 2023

In streaming/base/format/base/writer.py:

    @classmethod
    def _get_timer(cls) -> Timer:
        """Get a timer tree for the process of writing a dataset.

        Returns:
            Timer: The tree of timers.
        """
        return Timer([
            ('write', Timer([
                'serialize_sample',
                'flush_shard',
            ])),
            ('finish',
             Timer([
                 'flush_last_shard',
                 ('save_index',
                  Timer([
                      'serialize_index',
                      'write_index',
                      'wait_for_shard_uploads',
                      'upload_index',
                  ])),
                 'shutdown_executor',
                 'remove_local',
             ])),
        ])

Example output:

mds:
{
    "named_timers": [
        [
            "write",
            {
                "named_timers": [
                    [
                        "serialize_sample",
                        {
                            "stats": {
                                "count": 33554432,
                                "frac_of_parts": 0.8812000213489666,
                                "frac_of_whole": 0.7268265642055434,
                                "max": 2.171626367,
                                "mean": 1.8077239529788458e-06,
                                "min": 1.276e-06,
                                "quantiles": [
                                    1.276e-06,
                                    1.457e-06,
                                    1.49e-06,
                                    1.517e-06,
                                    1.544e-06,
                                    1.573e-06,
                                    1.606e-06,
                                    1.65e-06,
                                    1.722e-06,
                                    2.131e-06,
                                    2.171626367
                                ],
                                "std": 0.00041819769639734895,
                                "total": 60.65715045499988
                            }
                        }
                    ],
                    [
                        "flush_shard",
                        {
                            "stats": {
                                "count": 33554432,
                                "frac_of_parts": 0.11879997865103349,
                                "frac_of_whole": 0.09798794623091374,
                                "max": 0.018126742,
                                "mean": 2.4371035158634205e-07,
                                "min": 1.16e-07,
                                "quantiles": [
                                    1.16e-07,
                                    1.31e-07,
                                    1.34e-07,
                                    1.39e-07,
                                    1.43e-07,
                                    1.47e-07,
                                    1.5e-07,
                                    1.55e-07,
                                    1.6e-07,
                                    1.7e-07,
                                    0.018126742
                                ],
                                "std": 2.810335168383885e-05,
                                "total": 8.177562420000006
                            }
                        }
                    ]
                ],
                "stats": {
                    "count": 33554432,
                    "frac_of_parts": 0.9999638519330477,
                    "frac_of_whole": 0.8296981285330818,
                    "max": 2.171631721,
                    "mean": 2.4871462354362016e-06,
                    "min": 1.766e-06,
                    "quantiles": [
                        1.766e-06,
                        1.987e-06,
                        2.026e-06,
                        2.059e-06,
                        2.094e-06,
                        2.131e-06,
                        2.175e-06,
                        2.233e-06,
                        2.335e-06,
                        2.855e-06,
                        2.171631721
                    ],
                    "std": 0.00041914432833075374,
                    "total": 83.45477923100002
                }
            }
        ],
        [
            "finish",
            {
                "named_timers": [
                    [
                        "flush_last_shard",
                        {
                            "stats": {
                                "frac_of_parts": 0.6023615745512895,
                                "frac_of_whole": 0.6012550226429129,
                                "total": 0.001813889
                            }
                        }
                    ],
                    [
                        "save_index",
                        {
                            "named_timers": [
                                [
                                    "serialize_index",
                                    {
                                        "stats": {
                                            "frac_of_parts": 0.7763794708713782,
                                            "frac_of_whole": 0.7714316449183856,
                                            "total": 0.000824699
                                        }
                                    }
                                ],
                                [
                                    "write_index",
                                    {
                                        "stats": {
                                            "frac_of_parts": 0.20490719114472572,
                                            "frac_of_whole": 0.20360132828211963,
                                            "total": 0.00021766
                                        }
                                    }
                                ],
                                [
                                    "wait_for_shard_uploads",
                                    {
                                        "stats": {
                                            "frac_of_parts": 0.0009743588295267441,
                                            "frac_of_whole": 0.0009681492914269679,
                                            "total": 1.035e-06
                                        }
                                    }
                                ],
                                [
                                    "upload_index",
                                    {
                                        "stats": {
                                            "frac_of_parts": 0.017738979154369505,
                                            "frac_of_whole": 0.01762592956363126,
                                            "total": 1.8843e-05
                                        }
                                    }
                                ]
                            ],
                            "stats": {
                                "frac_of_parts": 0.3550132567505818,
                                "frac_of_whole": 0.3543610893259764,
                                "total": 0.00106905
                            }
                        }
                    ],
                    [
                        "shutdown_executor",
                        {
                            "stats": {
                                "frac_of_parts": 0.04247606346237633,
                                "frac_of_whole": 0.042398033968015524,
                                "total": 0.000127908
                            }
                        }
                    ],
                    [
                        "remove_local",
                        {
                            "stats": {
                                "frac_of_parts": 0.00014910523575231395,
                                "frac_of_whole": 0.00014883132604402358,
                                "total": 4.49e-07
                            }
                        }
                    ]
                ],
                "stats": {
                    "frac_of_parts": 3.614806695237654e-05,
                    "frac_of_whole": 2.9993067691894386e-05,
                    "total": 0.003016838
                }
            }
        ]
    ],
    "stats": {
        "total": 100.584509427
    }
}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

Successfully merging this pull request may close these issues.

1 participant