| /* |
| * Copyright (c) 2014-2015, Intel Corporation |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions are met: |
| * |
| * * Redistributions of source code must retain the above copyright notice, |
| * this list of conditions and the following disclaimer. |
| * * Redistributions in binary form must reproduce the above copyright notice, |
| * this list of conditions and the following disclaimer in the documentation |
| * and/or other materials provided with the distribution. |
| * * Neither the name of Intel Corporation nor the names of its contributors |
| * may be used to endorse or promote products derived from this software |
| * without specific prior written permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE |
| * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| * POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| #include "pt_time.h" |
| |
| #include "intel-pt.h" |
| |
| #include <string.h> |
| #include <limits.h> |
| |
| |
| void pt_time_init(struct pt_time *time) |
| { |
| if (!time) |
| return; |
| |
| memset(time, 0, sizeof(*time)); |
| } |
| |
| int pt_time_query_tsc(uint64_t *tsc, uint32_t *lost_mtc, |
| uint32_t *lost_cyc, const struct pt_time *time) |
| { |
| if (!tsc || !time) |
| return -pte_internal; |
| |
| *tsc = time->tsc; |
| |
| if (lost_mtc) |
| *lost_mtc = time->lost_mtc; |
| if (lost_cyc) |
| *lost_cyc = time->lost_cyc; |
| |
| if (!time->have_tsc) |
| return -pte_no_time; |
| |
| return 0; |
| } |
| |
| int pt_time_query_cbr(uint32_t *cbr, const struct pt_time *time) |
| { |
| if (!cbr || !time) |
| return -pte_internal; |
| |
| if (!time->have_cbr) |
| return -pte_no_cbr; |
| |
| *cbr = time->cbr; |
| |
| return 0; |
| } |
| |
| /* Compute the distance between two CTC sources. |
| * |
| * We adjust a single wrap-around but fail if the distance is bigger than that. |
| * |
| * Returns zero on success, a negative error code otherwise. |
| */ |
| static int pt_time_ctc_delta(uint32_t *ctc_delta, uint32_t ctc, |
| uint32_t last_ctc, const struct pt_config *config) |
| { |
| if (!config || !ctc_delta) |
| return -pte_internal; |
| |
| /* Correct a single wrap-around. If we lost enough MTCs to wrap |
| * around twice, timing will be wrong until the next TSC. |
| */ |
| if (ctc < last_ctc) { |
| ctc += 1u << (config->mtc_freq + pt_pl_mtc_bit_size); |
| |
| /* Since we only store the CTC between TMA/MTC or MTC/TMC a |
| * single correction should suffice. |
| */ |
| if (ctc < last_ctc) |
| return -pte_bad_packet; |
| } |
| |
| *ctc_delta = ctc - last_ctc; |
| return 0; |
| } |
| |
| /* Translate CTC into the same unit as the FastCounter by multiplying with P. |
| * |
| * Returns zero on success, a negative error code otherwise. |
| */ |
| static int pt_time_ctc_fc(uint64_t *fc, uint64_t ctc, |
| const struct pt_config *config) |
| { |
| uint32_t eax, ebx; |
| |
| if (!fc || !config) |
| return -pte_internal; |
| |
| eax = config->cpuid_0x15_eax; |
| ebx = config->cpuid_0x15_ebx; |
| |
| /* Neither multiply nor divide by zero. */ |
| if (!eax || !ebx) |
| return -pte_bad_config; |
| |
| *fc = (ctc * ebx) / eax; |
| return 0; |
| } |
| |
| int pt_time_update_tsc(struct pt_time *time, |
| const struct pt_packet_tsc *packet, |
| const struct pt_config *config) |
| { |
| (void) config; |
| |
| if (!time || !packet) |
| return -pte_internal; |
| |
| time->have_tsc = 1; |
| time->have_tma = 0; |
| time->tsc = time->base = packet->tsc; |
| time->fc = 0ull; |
| |
| /* We got the full time; we recover from previous losses. */ |
| time->lost_mtc = 0; |
| time->lost_cyc = 0; |
| |
| return 0; |
| } |
| |
| int pt_time_update_cbr(struct pt_time *time, |
| const struct pt_packet_cbr *packet, |
| const struct pt_config *config) |
| { |
| (void) config; |
| |
| if (!time || !packet) |
| return -pte_internal; |
| |
| time->have_cbr = 1; |
| time->cbr = packet->ratio; |
| |
| return 0; |
| } |
| |
| int pt_time_update_tma(struct pt_time *time, |
| const struct pt_packet_tma *packet, |
| const struct pt_config *config) |
| { |
| uint32_t ctc, mtc_mask, mtc_offset, mtc_freq; |
| uint64_t fc; |
| |
| if (!time || !packet || !config) |
| return -pte_internal; |
| |
| /* Without a TSC something is seriously wrong. */ |
| if (!time->have_tsc) |
| return -pte_bad_context; |
| |
| ctc = packet->ctc; |
| fc = packet->fc; |
| |
| mtc_freq = config->mtc_freq; |
| mtc_mask = (1u << mtc_freq) - 1u; |
| mtc_offset = ctc & mtc_mask; |
| |
| /* Mask out the MTC offset and the high order bits. */ |
| ctc &= pt_pl_mtc_mask << mtc_freq; |
| |
| time->have_tma = 1; |
| time->base -= fc; |
| time->fc += fc; |
| time->mtc_offset = mtc_offset; |
| |
| /* If the MTC frequency is low enough that TMA provides the full CTC |
| * value, we're fine. |
| */ |
| if ((mtc_freq + 8) <= pt_pl_tma_ctc_bit_size) { |
| time->ctc = ctc; |
| time->ctc_cyc = ctc; |
| |
| /* We can use the TMA instead of an MTC. */ |
| time->have_mtc = 1; |
| } |
| /* Otherwise, we must rely on either having seen an MTC before or |
| * on heuristically approximating it later at the next MTC. |
| */ |
| |
| return 0; |
| } |
| |
| int pt_time_update_mtc(struct pt_time *time, |
| const struct pt_packet_mtc *packet, |
| const struct pt_config *config) |
| { |
| uint32_t last_ctc, mtc_offset, ctc, ctc_delta; |
| uint64_t tsc, base; |
| uint8_t mtc_freq; |
| int errcode, have_tsc, have_tma, have_mtc; |
| |
| if (!time || !packet || !config) |
| return -pte_internal; |
| |
| have_tsc = time->have_tsc; |
| have_tma = time->have_tma; |
| have_mtc = time->have_mtc; |
| |
| /* We ignore MTCs between TSC and TMA to avoid apparent CTC overflows. |
| * Later MTCs will ensure that no time is lost. |
| */ |
| if (have_tsc && !have_tma) |
| return 0; |
| |
| base = time->base; |
| last_ctc = time->ctc; |
| mtc_offset = time->mtc_offset; |
| mtc_freq = config->mtc_freq; |
| |
| ctc = packet->ctc << mtc_freq; |
| |
| /* Store our CTC value if we have or would have reset FC. */ |
| if (time->fc || time->lost_cyc || !have_mtc) |
| time->ctc_cyc = ctc; |
| |
| /* Prepare for the next packet in case we error out below. */ |
| time->have_mtc = 1; |
| time->fc = 0ull; |
| time->mtc_offset = 0; |
| time->ctc = ctc; |
| |
| /* We recover from previous CYC losses. */ |
| time->lost_cyc = 0; |
| |
| /* Avoid a big jump when we see the first MTC with an arbitrary CTC |
| * payload. |
| */ |
| if (!have_mtc) { |
| uint8_t shift; |
| |
| /* If we have not seen a TMA, we ignore this first MTC. |
| * |
| * We have no idea where in this MTC period tracing started. |
| * We could lose an entire MTC period or just a tiny fraction. |
| * |
| * On the other hand, if we assumed a previous MTC value, we |
| * might make just the same error. |
| */ |
| if (!have_tma) |
| return 0; |
| |
| /* The TMA we've seen provided an offset into the current |
| * MTC period. Let's assume the last CTC was just small |
| * enough to contain that offset. |
| */ |
| shift = pt_pl_tma_ctc_bit_size; |
| if (shift < mtc_freq) |
| shift = mtc_freq; |
| |
| ctc_delta = 1u << shift; |
| } else { |
| /* This is the normal case. We have seen an MTC before so we |
| * know the previous CTC value. |
| */ |
| |
| errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config); |
| if (errcode < 0) { |
| time->lost_mtc += 1; |
| return errcode; |
| } |
| } |
| |
| /* We don't want a wrap-around here. Something must be wrong. */ |
| if (ctc_delta < mtc_offset) { |
| time->lost_mtc += 1; |
| return -pte_bad_packet; |
| } |
| |
| ctc_delta -= mtc_offset; |
| |
| errcode = pt_time_ctc_fc(&tsc, ctc_delta, config); |
| if (errcode < 0) |
| return errcode; |
| |
| base += tsc; |
| time->tsc = time->base = base; |
| |
| return 0; |
| } |
| |
| /* Adjust a CYC packet's payload spanning multiple MTC periods. |
| * |
| * CYC packets measure the Fast Counter since the last CYC(-eligible) packet. |
| * Depending on the CYC threshold, we may not get a CYC for each MTC, so a CYC |
| * period may overlap with or even span multiple MTC periods. |
| * |
| * We can't do much about the overlap case without examining all packets in |
| * the respective periods. We leave this as expected imprecision. |
| * |
| * If we find a CYC packet to span multiple MTC packets, though, we try to |
| * approximate the portion for the current MTC period by subtracting the |
| * estimated portion for previous MTC periods using calibration information. |
| * |
| * We only consider MTC. For the first CYC after TSC, the corresponding TMA |
| * will contain the Fast Counter at TSC. |
| * |
| * Returns zero on success, a negative error code otherwise. |
| */ |
| static int pt_time_adjust_cyc(uint64_t *cyc, const struct pt_time *time, |
| const struct pt_config *config, uint64_t fcr) |
| { |
| uint32_t last_ctc, ctc, ctc_delta; |
| uint64_t fc, total_cyc, old_cyc; |
| int errcode; |
| |
| if (!time || !config || !fcr) |
| return -pte_internal; |
| |
| last_ctc = time->ctc_cyc; |
| ctc = time->ctc; |
| |
| /* There is nothing to do if this is the current MTC period. */ |
| if (ctc == last_ctc) |
| return 0; |
| |
| /* Calibration computes |
| * |
| * fc = (ctc_delta * cpuid[0x15].ebx) / cpuid[0x15].eax. |
| * fcr = (fc << pt_tcal_fcr_shr) / cyc |
| * |
| * So cyc = (fc << pt_tcal_fcr_shr) / fcr. |
| */ |
| |
| errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config); |
| if (errcode < 0) |
| return errcode; |
| |
| errcode = pt_time_ctc_fc(&fc, ctc_delta, config); |
| if (errcode < 0) |
| return errcode; |
| |
| old_cyc = (fc << pt_tcal_fcr_shr) / fcr; |
| total_cyc = *cyc; |
| |
| /* Make sure we don't wrap around. If we would, attribute the entire |
| * CYC payload to any previous MTC period. |
| * |
| * We lost an unknown portion of the CYC payload for the current MTC |
| * period, but it's usually better to run too slow than too fast. |
| */ |
| if (total_cyc < old_cyc) |
| total_cyc = old_cyc; |
| |
| *cyc = total_cyc - old_cyc; |
| return 0; |
| } |
| |
| int pt_time_update_cyc(struct pt_time *time, |
| const struct pt_packet_cyc *packet, |
| const struct pt_config *config, uint64_t fcr) |
| { |
| uint64_t cyc, fc; |
| |
| if (!time || !packet || !config) |
| return -pte_internal; |
| |
| if (!fcr) { |
| time->lost_cyc += 1; |
| return 0; |
| } |
| |
| cyc = packet->value; |
| fc = time->fc; |
| if (!fc) { |
| int errcode; |
| |
| errcode = pt_time_adjust_cyc(&cyc, time, config, fcr); |
| if (errcode < 0) |
| return errcode; |
| } |
| |
| fc += (cyc * fcr) >> pt_tcal_fcr_shr; |
| |
| time->fc = fc; |
| time->tsc = time->base + fc; |
| |
| return 0; |
| } |
| |
| void pt_tcal_init(struct pt_time_cal *tcal) |
| { |
| if (!tcal) |
| return; |
| |
| memset(tcal, 0, sizeof(*tcal)); |
| |
| tcal->min_fcr = UINT64_MAX; |
| } |
| |
| static int pt_tcal_have_fcr(const struct pt_time_cal *tcal) |
| { |
| if (!tcal) |
| return 0; |
| |
| return (tcal->min_fcr <= tcal->max_fcr); |
| } |
| |
| int pt_tcal_fcr(uint64_t *fcr, const struct pt_time_cal *tcal) |
| { |
| if (!fcr || !tcal) |
| return -pte_internal; |
| |
| if (!pt_tcal_have_fcr(tcal)) |
| return -pte_no_time; |
| |
| *fcr = tcal->fcr; |
| |
| return 0; |
| } |
| |
| int pt_tcal_set_fcr(struct pt_time_cal *tcal, uint64_t fcr) |
| { |
| if (!tcal) |
| return -pte_internal; |
| |
| tcal->fcr = fcr; |
| |
| if (fcr < tcal->min_fcr) |
| tcal->min_fcr = fcr; |
| |
| if (fcr > tcal->max_fcr) |
| tcal->max_fcr = fcr; |
| |
| return 0; |
| } |
| |
| int pt_tcal_update_tsc(struct pt_time_cal *tcal, |
| const struct pt_packet_tsc *packet, |
| const struct pt_config *config) |
| { |
| (void) config; |
| |
| if (!tcal || !packet) |
| return -pte_internal; |
| |
| /* A TSC outside of PSB+ may indicate loss of time. We do not use it |
| * for calibration. We store the TSC value for calibration at the next |
| * TSC in PSB+, though. |
| */ |
| tcal->tsc = packet->tsc; |
| tcal->cyc_tsc = 0ull; |
| |
| return 0; |
| } |
| |
| int pt_tcal_header_tsc(struct pt_time_cal *tcal, |
| const struct pt_packet_tsc *packet, |
| const struct pt_config *config) |
| { |
| uint64_t tsc, last_tsc, tsc_delta, cyc, fcr; |
| |
| (void) config; |
| |
| if (!tcal || !packet) |
| return -pte_internal; |
| |
| last_tsc = tcal->tsc; |
| cyc = tcal->cyc_tsc; |
| |
| tsc = packet->tsc; |
| |
| tcal->tsc = tsc; |
| tcal->cyc_tsc = 0ull; |
| |
| if (!last_tsc || !cyc) |
| return 0; |
| |
| /* Correct a single wrap-around. */ |
| if (tsc < last_tsc) { |
| tsc += 1ull << pt_pl_tsc_bit_size; |
| |
| if (tsc < last_tsc) |
| return -pte_bad_packet; |
| } |
| |
| tsc_delta = tsc - last_tsc; |
| |
| /* We shift the nominator to improve rounding precision. |
| * |
| * Since we're only collecting the CYCs between two TSC, we shouldn't |
| * overflow. Let's rather fail than overflow. |
| */ |
| if (tsc_delta & ~(~0ull >> pt_tcal_fcr_shr)) |
| return -pte_internal; |
| |
| fcr = (tsc_delta << pt_tcal_fcr_shr) / cyc; |
| |
| return pt_tcal_set_fcr(tcal, fcr); |
| } |
| |
| int pt_tcal_update_cbr(struct pt_time_cal *tcal, |
| const struct pt_packet_cbr *packet, |
| const struct pt_config *config) |
| { |
| /* A CBR outside of PSB+ indicates a frequency change. Reset our |
| * calibration state. |
| */ |
| pt_tcal_init(tcal); |
| |
| return pt_tcal_header_cbr(tcal, packet, config); |
| } |
| |
| int pt_tcal_header_cbr(struct pt_time_cal *tcal, |
| const struct pt_packet_cbr *packet, |
| const struct pt_config *config) |
| { |
| uint64_t cbr, p1, fcr; |
| |
| if (!tcal || !packet || !config) |
| return -pte_internal; |
| |
| p1 = config->nom_freq; |
| if (!p1) |
| return 0; |
| |
| /* If we know the nominal frequency, we can use it for calibration. */ |
| cbr = packet->ratio; |
| |
| fcr = (p1 << pt_tcal_fcr_shr) / cbr; |
| |
| return pt_tcal_set_fcr(tcal, fcr); |
| } |
| |
| int pt_tcal_update_tma(struct pt_time_cal *tcal, |
| const struct pt_packet_tma *packet, |
| const struct pt_config *config) |
| { |
| (void) tcal; |
| (void) packet; |
| (void) config; |
| |
| /* Nothing to do. */ |
| return 0; |
| } |
| |
| int pt_tcal_update_mtc(struct pt_time_cal *tcal, |
| const struct pt_packet_mtc *packet, |
| const struct pt_config *config) |
| { |
| uint32_t last_ctc, ctc, ctc_delta, have_mtc; |
| uint64_t cyc, fc, fcr; |
| int errcode; |
| |
| if (!tcal || !packet || !config) |
| return -pte_internal; |
| |
| last_ctc = tcal->ctc; |
| have_mtc = tcal->have_mtc; |
| cyc = tcal->cyc_mtc; |
| |
| ctc = packet->ctc << config->mtc_freq; |
| |
| /* We need at least two MTC (including this). */ |
| if (!have_mtc) { |
| tcal->cyc_mtc = 0ull; |
| tcal->ctc = ctc; |
| tcal->have_mtc = 1; |
| |
| return 0; |
| } |
| |
| /* Without any cycles, we can't calibrate. Try again at the next |
| * MTC and distribute the cycles over the combined MTC period. |
| */ |
| if (!cyc) |
| return 0; |
| |
| /* Prepare for the next packet in case we error out below. */ |
| tcal->have_mtc = 1; |
| tcal->cyc_mtc = 0ull; |
| tcal->ctc = ctc; |
| |
| /* Let's pretend we will fail. We'll correct it at the end. */ |
| tcal->lost_mtc += 1; |
| |
| errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config); |
| if (errcode < 0) |
| return errcode; |
| |
| errcode = pt_time_ctc_fc(&fc, ctc_delta, config); |
| if (errcode < 0) |
| return errcode; |
| |
| /* We shift the nominator to improve rounding precision. |
| * |
| * Since we're only collecting the CYCs between two MTC, we shouldn't |
| * overflow. Let's rather fail than overflow. |
| */ |
| if (fc & ~(~0ull >> pt_tcal_fcr_shr)) |
| return -pte_internal; |
| |
| fcr = (fc << pt_tcal_fcr_shr) / cyc; |
| |
| errcode = pt_tcal_set_fcr(tcal, fcr); |
| if (errcode < 0) |
| return errcode; |
| |
| /* We updated the FCR. This recovers from previous MTC losses. */ |
| tcal->lost_mtc = 0; |
| |
| return 0; |
| } |
| |
| int pt_tcal_update_cyc(struct pt_time_cal *tcal, |
| const struct pt_packet_cyc *packet, |
| const struct pt_config *config) |
| { |
| uint64_t cyc; |
| |
| (void) config; |
| |
| if (!tcal || !packet) |
| return -pte_internal; |
| |
| cyc = packet->value; |
| tcal->cyc_mtc += cyc; |
| tcal->cyc_tsc += cyc; |
| |
| return 0; |
| } |