Latency Analyzer

WITH 1 AS last_hours_to_query
SELECT
    normalized_query_hash as group,
    query_duration_ms * 1000 as _duration,
    intDiv(ProfileEvents['LoggerElapsedNanoseconds'], 1000) as LoggerElapsedNanoseconds,
    ProfileEvents['GlobalThreadPoolLockWaitMicroseconds'] as GlobalThreadPoolLockWait,
    ProfileEvents['LocalThreadPoolLockWaitMicroseconds'] as LocalThreadPoolLockWait,
    ProfileEvents['SchedulerIOReadWaitMicroseconds'] as SchedulerIOReadWait,
    ProfileEvents['SchedulerIOWriteWaitMicroseconds'] as SchedulerIOWriteWait,
    ProfileEvents['ZooKeeperWaitMicroseconds'] as ZooKeeperWait,
    ProfileEvents['ContextLockWaitMicroseconds'] as ContextLockWait,
    ProfileEvents['PartsLockWaitMicroseconds'] as PartsLockWait,
    ProfileEvents['AsynchronousReadWaitMicroseconds'] as AsynchronousReadWait,
    ProfileEvents['SynchronousReadWaitMicroseconds'] as SynchronousReadWait,
    ProfileEvents['AsynchronousRemoteReadWaitMicroseconds'] as AsynchronousRemoteReadWait,
    ProfileEvents['SynchronousRemoteReadWaitMicroseconds'] as SynchronousRemoteReadWait,
    ProfileEvents['AsyncLoaderWaitMicroseconds'] as AsyncLoaderWait,
    ProfileEvents['ConcurrencyControlWaitMicroseconds'] as ConcurrencyControlWait,
    ProfileEvents['FileSyncElapsedMicroseconds'] as FileSyncElapsed,
    ProfileEvents['DirectorySyncElapsedMicroseconds'] as DirectorySyncElapsed,
    ProfileEvents['CompressedReadBufferChecksumDoesntMatchMicroseconds'] as CompressedReadBufferChecksumDoesntMatch,
    ProfileEvents['OpenedFileCacheMicroseconds'] as OpenedFileCache,
    ProfileEvents['DiskReadElapsedMicroseconds'] as DiskReadElapsed,
    ProfileEvents['DiskWriteElapsedMicroseconds'] as DiskWriteElapsed,
    ProfileEvents['NetworkReceiveElapsedMicroseconds'] as NetworkReceiveElapsed,
    ProfileEvents['NetworkSendElapsedMicroseconds'] as NetworkSendElapsed,
    ProfileEvents['GlobalThreadPoolThreadCreationMicroseconds'] as GlobalThreadPoolThreadCreation,
    ProfileEvents['LocalThreadPoolThreadCreationMicroseconds'] as LocalThreadPoolThreadCreation,
    ProfileEvents['DiskS3GetRequestThrottlerSleepMicroseconds'] as DiskS3GetRequestThrottlerSleep,
    ProfileEvents['DiskS3PutRequestThrottlerSleepMicroseconds'] as DiskS3PutRequestThrottlerSleep,
    ProfileEvents['S3GetRequestThrottlerSleepMicroseconds'] as S3GetRequestThrottlerSleep,
    ProfileEvents['S3PutRequestThrottlerSleepMicroseconds'] as S3PutRequestThrottlerSleep,
    ProfileEvents['RemoteReadThrottlerSleepMicroseconds'] as RemoteReadThrottlerSleep,
    ProfileEvents['RemoteWriteThrottlerSleepMicroseconds'] as RemoteWriteThrottlerSleep,
    ProfileEvents['LocalReadThrottlerSleepMicroseconds'] as LocalReadThrottlerSleep,
    ProfileEvents['LocalWriteThrottlerSleepMicroseconds'] as LocalWriteThrottlerSleep,
    ProfileEvents['CompileExpressionsMicroseconds'] as CompileExpressions,
    ProfileEvents['FilteringMarksWithPrimaryKeyMicroseconds'] as FilteringMarksWithPrimaryKey,
    ProfileEvents['FilteringMarksWithSecondaryKeysMicroseconds'] as FilteringMarksWithSecondaryKeys,
    ProfileEvents['WaitMarksLoadMicroseconds'] as WaitMarksLoad,
    ProfileEvents['MemoryOvercommitWaitTimeMicroseconds'] as MemoryOvercommitWaitTime,
    ProfileEvents['MemoryAllocatorPurgeTimeMicroseconds'] as MemoryAllocatorPurgeTime,
    ProfileEvents['ParallelReplicasHandleRequestMicroseconds'] as ParallelReplicasHandleRequest,
    ProfileEvents['ParallelReplicasHandleAnnouncementMicroseconds'] as ParallelReplicasHandleAnnouncement,
    ProfileEvents['ParallelReplicasAnnouncementMicroseconds'] as ParallelReplicasAnnouncement,
    ProfileEvents['ParallelReplicasReadRequestMicroseconds'] as ParallelReplicasReadRequest,
    ProfileEvents['ParallelReplicasStealingByHashMicroseconds'] as ParallelReplicasStealingByHash,
    ProfileEvents['ParallelReplicasProcessingPartsMicroseconds'] as ParallelReplicasProcessingParts,
    ProfileEvents['ParallelReplicasStealingLeftoversMicroseconds'] as ParallelReplicasStealingLeftovers,
    ProfileEvents['ParallelReplicasCollectingOwnedSegmentsMicroseconds'] as ParallelReplicasCollectingOwnedSegments,
    ProfileEvents['S3ReadMicroseconds'] as S3Read,
    ProfileEvents['S3WriteMicroseconds'] as S3Write,
    ProfileEvents['DiskS3ReadMicroseconds'] as DiskS3Read,
    ProfileEvents['DiskS3WriteMicroseconds'] as DiskS3Write,
    ProfileEvents['ReadBufferFromS3Microseconds'] as ReadBufferFromS3,
    ProfileEvents['ReadBufferFromS3InitMicroseconds'] as ReadBufferFromS3Init,
    ProfileEvents['WriteBufferFromS3Microseconds'] as WriteBufferFromS3,
    ProfileEvents['WriteBufferFromS3WaitInflightLimitMicroseconds'] as WriteBufferFromS3WaitInflightLimit,
    ProfileEvents['ReadBufferFromAzureMicroseconds'] as ReadBufferFromAzure,
    ProfileEvents['ReadBufferFromAzureInitMicroseconds'] as ReadBufferFromAzureInit,
    ProfileEvents['CachedReadBufferReadFromSourceMicroseconds'] as CachedReadBufferReadFromSource,
    ProfileEvents['CachedReadBufferReadFromCacheMicroseconds'] as CachedReadBufferReadFromCache,
    ProfileEvents['CachedReadBufferCacheWriteMicroseconds'] as CachedReadBufferCacheWrite,
    ProfileEvents['CachedReadBufferCreateBufferMicroseconds'] as CachedReadBufferCreateBuffer,
    ProfileEvents['CachedWriteBufferCacheWriteMicroseconds'] as CachedWriteBufferCacheWrite,
    ProfileEvents['FilesystemCacheLoadMetadataMicroseconds'] as FilesystemCacheLoadMetadata,
    ProfileEvents['FilesystemCacheLockKeyMicroseconds'] as FilesystemCacheLockKey,
    ProfileEvents['FilesystemCacheLockMetadataMicroseconds'] as FilesystemCacheLockMetadata,
    ProfileEvents['FilesystemCacheLockCacheMicroseconds'] as FilesystemCacheLockCache,
    ProfileEvents['FilesystemCacheReserveMicroseconds'] as FilesystemCacheReserve,
    ProfileEvents['FilesystemCacheEvictMicroseconds'] as FilesystemCacheEvict,
    ProfileEvents['FilesystemCacheGetOrSetMicroseconds'] as FilesystemCacheGetOrSet,
    ProfileEvents['FilesystemCacheGetMicroseconds'] as FilesystemCacheGet,
    ProfileEvents['FileSegmentWaitMicroseconds'] as FileSegmentWait,
    ProfileEvents['FileSegmentCompleteMicroseconds'] as FileSegmentComplete,
    ProfileEvents['FileSegmentLockMicroseconds'] as FileSegmentLock,
    ProfileEvents['FileSegmentWriteMicroseconds'] as FileSegmentWrite,
    ProfileEvents['FileSegmentUseMicroseconds'] as FileSegmentUse,
    ProfileEvents['FileSegmentRemoveMicroseconds'] as FileSegmentRemove,
    ProfileEvents['FileSegmentHolderCompleteMicroseconds'] as FileSegmentHolderComplete,
    ProfileEvents['WaitPrefetchTaskMicroseconds'] as WaitPrefetchTask,
    ProfileEvents['ThreadpoolReaderTaskMicroseconds'] as ThreadpoolReaderTask,
    ProfileEvents['ThreadpoolReaderPrepareMicroseconds'] as ThreadpoolReaderPrepare,
    ProfileEvents['ThreadpoolReaderSubmitReadSynchronouslyMicroseconds'] as ThreadpoolReaderSubmitReadSynchronously,
    ProfileEvents['ThreadpoolReaderSubmitLookupInCacheMicroseconds'] as ThreadpoolReaderSubmitLookupInCache,
    ProfileEvents['SleepFunctionElapsedMicroseconds'] as SleepFunctionElapsed,
    ProfileEvents['ThreadPoolReaderPageCacheHitElapsedMicroseconds'] as ThreadPoolReaderPageCacheHitElapsed,
    ProfileEvents['ThreadPoolReaderPageCacheMissElapsedMicroseconds'] as ThreadPoolReaderPageCacheMissElapsed,
    ProfileEvents['ConnectionPoolIsFullMicroseconds'] as ConnectionPoolIsFull,
    ProfileEvents['SharedMergeTreeVirtualPartsUpdateMicroseconds'] as SharedMergeTreeVirtualPartsUpdate,
    ProfileEvents['SharedMergeTreeVirtualPartsUpdatesFromZooKeeperMicroseconds'] as SharedMergeTreeVirtualPartsUpdatesFromZooKeeper,
    ProfileEvents['SharedMergeTreeVirtualPartsUpdatesFromPeerMicroseconds'] as SharedMergeTreeVirtualPartsUpdatesFromPeer,
    ProfileEvents['SharedMergeTreeMergeSelectingTaskMicroseconds'] as SharedMergeTreeMergeSelectingTask,
    ProfileEvents['SharedMergeTreeScheduleDataProcessingJobMicroseconds'] as SharedMergeTreeScheduleDataProcessingJob,
    ProfileEvents['SharedMergeTreeHandleFetchPartsMicroseconds'] as SharedMergeTreeHandleFetchParts,
    ProfileEvents['SharedMergeTreeHandleOutdatedPartsMicroseconds'] as SharedMergeTreeHandleOutdatedParts,
    ProfileEvents['SharedMergeTreeGetPartsBatchToLoadMicroseconds'] as SharedMergeTreeGetPartsBatchToLoad,
    ProfileEvents['SharedMergeTreeTryUpdateDiskMetadataCacheForPartMicroseconds'] as SharedMergeTreeTryUpdateDiskMetadataCacheForPart,
    ProfileEvents['SharedMergeTreeLoadChecksumAndIndexesMicroseconds'] as SharedMergeTreeLoadChecksumAndIndexes,
    ProfileEvents['StorageConnectionsElapsedMicroseconds'] as StorageConnectionsElapsed,
    ProfileEvents['DiskConnectionsElapsedMicroseconds'] as DiskConnectionsElapsed,
    ProfileEvents['HTTPConnectionsElapsedMicroseconds'] as HTTPConnectionsElapsed,
    ProfileEvents['ParquetFetchWaitTimeMicroseconds'] as ParquetFetchWaitTime,
    ProfileEvents['OSCPUWaitMicroseconds'] as OSCPUWait,
    ProfileEvents['OSIOWaitMicroseconds'] as OSIOWait,
    ProfileEvents['OSCPUVirtualTimeMicroseconds'] as OSCPUVirtualTimeMicroseconds
FROM system.query_log
WHERE event_time >= now() - INTERVAL last_hours_to_query HOUR
  AND type = 2
SETTINGS output_format_json_named_tuples_as_objects = 1, skip_unavailable_shards = 1

Run SYSTEM FLUSH LOGS

Filter insignificant columns