libipt/src/pt_time.c - third_party/processor-trace - Git at Google

 /*
  * Copyright (c) 2014-2015, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  * Redistributions of source code must retain the above copyright notice,
  *    this list of conditions and the following disclaimer.
  *  * Redistributions in binary form must reproduce the above copyright notice,
  *    this list of conditions and the following disclaimer in the documentation
  *    and/or other materials provided with the distribution.
  *  * Neither the name of Intel Corporation nor the names of its contributors
  *    may be used to endorse or promote products derived from this software
  *    without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */

 #include "pt_time.h"

 #include "intel-pt.h"

 #include <string.h>
 #include <limits.h>


 void pt_time_init(struct pt_time *time)
 {
 	if (!time)
 		return;

 	memset(time, 0, sizeof(*time));
 }

 int pt_time_query_tsc(uint64_t *tsc, uint32_t *lost_mtc,
 		      uint32_t *lost_cyc, const struct pt_time *time)
 {
 	if (!tsc || !time)
 		return -pte_internal;

 	*tsc = time->tsc;

 	if (lost_mtc)
 		*lost_mtc = time->lost_mtc;
 	if (lost_cyc)
 		*lost_cyc = time->lost_cyc;

 	if (!time->have_tsc)
 		return -pte_no_time;

 	return 0;
 }

 int pt_time_query_cbr(uint32_t *cbr, const struct pt_time *time)
 {
 	if (!cbr || !time)
 		return -pte_internal;

 	if (!time->have_cbr)
 		return -pte_no_cbr;

 	*cbr = time->cbr;

 	return 0;
 }

 /* Compute the distance between two CTC sources.
  *
  * We adjust a single wrap-around but fail if the distance is bigger than that.
  *
  * Returns zero on success, a negative error code otherwise.
  */
 static int pt_time_ctc_delta(uint32_t *ctc_delta, uint32_t ctc,
 			     uint32_t last_ctc, const struct pt_config *config)
 {
 	if (!config || !ctc_delta)
 		return -pte_internal;

 	/* Correct a single wrap-around.  If we lost enough MTCs to wrap
 	 * around twice, timing will be wrong until the next TSC.
 	 */
 	if (ctc < last_ctc) {
 		ctc += 1u << (config->mtc_freq + pt_pl_mtc_bit_size);

 		/* Since we only store the CTC between TMA/MTC or MTC/TMC a
 		 * single correction should suffice.
 		 */
 		if (ctc < last_ctc)
 			return -pte_bad_packet;
 	}

 	*ctc_delta = ctc - last_ctc;
 	return 0;
 }

 /* Translate CTC into the same unit as the FastCounter by multiplying with P.
  *
  * Returns zero on success, a negative error code otherwise.
  */
 static int pt_time_ctc_fc(uint64_t *fc, uint64_t ctc,
 			  const struct pt_config *config)
 {
 	uint32_t eax, ebx;

 	if (!fc || !config)
 		return -pte_internal;

 	eax = config->cpuid_0x15_eax;
 	ebx = config->cpuid_0x15_ebx;

 	/* Neither multiply nor divide by zero. */
 	if (!eax || !ebx)
 		return -pte_bad_config;

 	*fc = (ctc * ebx) / eax;
 	return 0;
 }

 int pt_time_update_tsc(struct pt_time *time,
 		       const struct pt_packet_tsc *packet,
 		       const struct pt_config *config)
 {
 	(void) config;

 	if (!time || !packet)
 		return -pte_internal;

 	time->have_tsc = 1;
 	time->have_tma = 0;
 	time->tsc = time->base = packet->tsc;
 	time->fc = 0ull;

 	/* We got the full time; we recover from previous losses. */
 	time->lost_mtc = 0;
 	time->lost_cyc = 0;

 	return 0;
 }

 int pt_time_update_cbr(struct pt_time *time,
 		       const struct pt_packet_cbr *packet,
 		       const struct pt_config *config)
 {
 	(void) config;

 	if (!time || !packet)
 		return -pte_internal;

 	time->have_cbr = 1;
 	time->cbr = packet->ratio;

 	return 0;
 }

 int pt_time_update_tma(struct pt_time *time,
 		       const struct pt_packet_tma *packet,
 		       const struct pt_config *config)
 {
 	uint32_t ctc, mtc_mask, mtc_offset, mtc_freq;
 	uint64_t fc;

 	if (!time || !packet || !config)
 		return -pte_internal;

 	/* Without a TSC something is seriously wrong. */
 	if (!time->have_tsc)
 		return -pte_bad_context;

 	ctc = packet->ctc;
 	fc = packet->fc;

 	mtc_freq = config->mtc_freq;
 	mtc_mask = (1u << mtc_freq) - 1u;
 	mtc_offset = ctc & mtc_mask;

 	/* Mask out the MTC offset and the high order bits. */
 	ctc &= pt_pl_mtc_mask << mtc_freq;

 	time->have_tma = 1;
 	time->base -= fc;
 	time->fc += fc;
 	time->mtc_offset = mtc_offset;

 	/* If the MTC frequency is low enough that TMA provides the full CTC
 	 * value, we're fine.
 	 */
 	if ((mtc_freq + 8) <= pt_pl_tma_ctc_bit_size) {
 		time->ctc = ctc;
 		time->ctc_cyc = ctc;

 		/* We can use the TMA instead of an MTC. */
 		time->have_mtc = 1;
 	}
 	/* Otherwise, we must rely on either having seen an MTC before or
 	 * on heuristically approximating it later at the next MTC.
 	 */

 	return 0;
 }

 int pt_time_update_mtc(struct pt_time *time,
 		       const struct pt_packet_mtc *packet,
 		       const struct pt_config *config)
 {
 	uint32_t last_ctc, mtc_offset, ctc, ctc_delta;
 	uint64_t tsc, base;
 	uint8_t mtc_freq;
 	int errcode, have_tsc, have_tma, have_mtc;

 	if (!time || !packet || !config)
 		return -pte_internal;

 	have_tsc = time->have_tsc;
 	have_tma = time->have_tma;
 	have_mtc = time->have_mtc;

 	/* We ignore MTCs between TSC and TMA to avoid apparent CTC overflows.
 	 * Later MTCs will ensure that no time is lost.
 	 */
 	if (have_tsc && !have_tma)
 		return 0;

 	base = time->base;
 	last_ctc = time->ctc;
 	mtc_offset = time->mtc_offset;
 	mtc_freq = config->mtc_freq;

 	ctc = packet->ctc << mtc_freq;

 	/* Store our CTC value if we have or would have reset FC. */
 	if (time->fc || time->lost_cyc || !have_mtc)
 		time->ctc_cyc = ctc;

 	/* Prepare for the next packet in case we error out below. */
 	time->have_mtc = 1;
 	time->fc = 0ull;
 	time->mtc_offset = 0;
 	time->ctc = ctc;

 	/* We recover from previous CYC losses. */
 	time->lost_cyc = 0;

 	/* Avoid a big jump when we see the first MTC with an arbitrary CTC
 	 * payload.
 	 */
 	if (!have_mtc) {
 		uint8_t shift;

 		/* If we have not seen a TMA, we ignore this first MTC.
 		 *
 		 * We have no idea where in this MTC period tracing started.
 		 * We could lose an entire MTC period or just a tiny fraction.
 		 *
 		 * On the other hand, if we assumed a previous MTC value, we
 		 * might make just the same error.
 		 */
 		if (!have_tma)
 			return 0;

 		/* The TMA we've seen provided an offset into the current
 		 * MTC period.  Let's assume the last CTC was just small
 		 * enough to contain that offset.
 		 */
 		shift = pt_pl_tma_ctc_bit_size;
 		if (shift < mtc_freq)
 			shift = mtc_freq;

 		ctc_delta = 1u << shift;
 	} else {
 		/* This is the normal case.  We have seen an MTC before so we
 		 * know the previous CTC value.
 		 */

 		errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
 		if (errcode < 0) {
 			time->lost_mtc += 1;
 			return errcode;
 		}
 	}

 	/* We don't want a wrap-around here.  Something must be wrong. */
 	if (ctc_delta < mtc_offset) {
 		time->lost_mtc += 1;
 		return -pte_bad_packet;
 	}

 	ctc_delta -= mtc_offset;

 	errcode = pt_time_ctc_fc(&tsc, ctc_delta, config);
 	if (errcode < 0)
 		return errcode;

 	base += tsc;
 	time->tsc = time->base = base;

 	return 0;
 }

 /* Adjust a CYC packet's payload spanning multiple MTC periods.
  *
  * CYC packets measure the Fast Counter since the last CYC(-eligible) packet.
  * Depending on the CYC threshold, we may not get a CYC for each MTC, so a CYC
  * period may overlap with or even span multiple MTC periods.
  *
  * We can't do much about the overlap case without examining all packets in
  * the respective periods.  We leave this as expected imprecision.
  *
  * If we find a CYC packet to span multiple MTC packets, though, we try to
  * approximate the portion for the current MTC period by subtracting the
  * estimated portion for previous MTC periods using calibration information.
  *
  * We only consider MTC.  For the first CYC after TSC, the corresponding TMA
  * will contain the Fast Counter at TSC.
  *
  * Returns zero on success, a negative error code otherwise.
  */
 static int pt_time_adjust_cyc(uint64_t *cyc, const struct pt_time *time,
 			      const struct pt_config *config, uint64_t fcr)
 {
 	uint32_t last_ctc, ctc, ctc_delta;
 	uint64_t fc, total_cyc, old_cyc;
 	int errcode;

 	if (!time || !config || !fcr)
 		return -pte_internal;

 	last_ctc = time->ctc_cyc;
 	ctc = time->ctc;

 	/* There is nothing to do if this is the current MTC period. */
 	if (ctc == last_ctc)
 		return 0;

 	/* Calibration computes
 	 *
 	 *   fc  = (ctc_delta * cpuid[0x15].ebx) / cpuid[0x15].eax.
 	 *   fcr = (fc << pt_tcal_fcr_shr) / cyc
 	 *
 	 * So cyc = (fc << pt_tcal_fcr_shr) / fcr.
 	 */

 	errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
 	if (errcode < 0)
 		return errcode;

 	errcode = pt_time_ctc_fc(&fc, ctc_delta, config);
 	if (errcode < 0)
 		return errcode;

 	old_cyc = (fc << pt_tcal_fcr_shr) / fcr;
 	total_cyc = *cyc;

 	/* Make sure we don't wrap around.  If we would, attribute the entire
 	 * CYC payload to any previous MTC period.
 	 *
 	 * We lost an unknown portion of the CYC payload for the current MTC
 	 * period, but it's usually better to run too slow than too fast.
 	 */
 	if (total_cyc < old_cyc)
 		total_cyc = old_cyc;

 	*cyc = total_cyc - old_cyc;
 	return 0;
 }

 int pt_time_update_cyc(struct pt_time *time,
 		       const struct pt_packet_cyc *packet,
 		       const struct pt_config *config, uint64_t fcr)
 {
 	uint64_t cyc, fc;

 	if (!time || !packet || !config)
 		return -pte_internal;

 	if (!fcr) {
 		time->lost_cyc += 1;
 		return 0;
 	}

 	cyc = packet->value;
 	fc = time->fc;
 	if (!fc) {
 		int errcode;

 		errcode = pt_time_adjust_cyc(&cyc, time, config, fcr);
 		if (errcode < 0)
 			return errcode;
 	}

 	fc += (cyc * fcr) >> pt_tcal_fcr_shr;

 	time->fc = fc;
 	time->tsc = time->base + fc;

 	return 0;
 }

 void pt_tcal_init(struct pt_time_cal *tcal)
 {
 	if (!tcal)
 		return;

 	memset(tcal, 0, sizeof(*tcal));

 	tcal->min_fcr = UINT64_MAX;
 }

 static int pt_tcal_have_fcr(const struct pt_time_cal *tcal)
 {
 	if (!tcal)
 		return 0;

 	return (tcal->min_fcr <= tcal->max_fcr);
 }

 int pt_tcal_fcr(uint64_t *fcr, const struct pt_time_cal *tcal)
 {
 	if (!fcr || !tcal)
 		return -pte_internal;

 	if (!pt_tcal_have_fcr(tcal))
 		return -pte_no_time;

 	*fcr = tcal->fcr;

 	return 0;
 }

 int pt_tcal_set_fcr(struct pt_time_cal *tcal, uint64_t fcr)
 {
 	if (!tcal)
 		return -pte_internal;

 	tcal->fcr = fcr;

 	if (fcr < tcal->min_fcr)
 		tcal->min_fcr = fcr;

 	if (fcr > tcal->max_fcr)
 		tcal->max_fcr = fcr;

 	return 0;
 }

 int pt_tcal_update_tsc(struct pt_time_cal *tcal,
 		      const struct pt_packet_tsc *packet,
 		      const struct pt_config *config)
 {
 	(void) config;

 	if (!tcal || !packet)
 		return -pte_internal;

 	/* A TSC outside of PSB+ may indicate loss of time.  We do not use it
 	 * for calibration.  We store the TSC value for calibration at the next
 	 * TSC in PSB+, though.
 	 */
 	tcal->tsc = packet->tsc;
 	tcal->cyc_tsc = 0ull;

 	return 0;
 }

 int pt_tcal_header_tsc(struct pt_time_cal *tcal,
 		      const struct pt_packet_tsc *packet,
 		      const struct pt_config *config)
 {
 	uint64_t tsc, last_tsc, tsc_delta, cyc, fcr;

 	(void) config;

 	if (!tcal || !packet)
 		return -pte_internal;

 	last_tsc = tcal->tsc;
 	cyc = tcal->cyc_tsc;

 	tsc = packet->tsc;

 	tcal->tsc = tsc;
 	tcal->cyc_tsc = 0ull;

 	if (!last_tsc || !cyc)
 		return 0;

 	/* Correct a single wrap-around. */
 	if (tsc < last_tsc) {
 		tsc += 1ull << pt_pl_tsc_bit_size;

 		if (tsc < last_tsc)
 			return -pte_bad_packet;
 	}

 	tsc_delta = tsc - last_tsc;

 	/* We shift the nominator to improve rounding precision.
 	 *
 	 * Since we're only collecting the CYCs between two TSC, we shouldn't
 	 * overflow.  Let's rather fail than overflow.
 	 */
 	if (tsc_delta & ~(~0ull >> pt_tcal_fcr_shr))
 		return -pte_internal;

 	fcr = (tsc_delta << pt_tcal_fcr_shr) / cyc;

 	return pt_tcal_set_fcr(tcal, fcr);
 }

 int pt_tcal_update_cbr(struct pt_time_cal *tcal,
 		      const struct pt_packet_cbr *packet,
 		      const struct pt_config *config)
 {
 	/* A CBR outside of PSB+ indicates a frequency change.  Reset our
 	 * calibration state.
 	 */
 	pt_tcal_init(tcal);

 	return pt_tcal_header_cbr(tcal, packet, config);
 }

 int pt_tcal_header_cbr(struct pt_time_cal *tcal,
 		      const struct pt_packet_cbr *packet,
 		      const struct pt_config *config)
 {
 	uint64_t cbr, p1, fcr;

 	if (!tcal || !packet || !config)
 		return -pte_internal;

 	p1 = config->nom_freq;
 	if (!p1)
 		return 0;

 	/* If we know the nominal frequency, we can use it for calibration. */
 	cbr = packet->ratio;

 	fcr = (p1 << pt_tcal_fcr_shr) / cbr;

 	return pt_tcal_set_fcr(tcal, fcr);
 }

 int pt_tcal_update_tma(struct pt_time_cal *tcal,
 		      const struct pt_packet_tma *packet,
 		      const struct pt_config *config)
 {
 	(void) tcal;
 	(void) packet;
 	(void) config;

 	/* Nothing to do. */
 	return 0;
 }

 int pt_tcal_update_mtc(struct pt_time_cal *tcal,
 		      const struct pt_packet_mtc *packet,
 		      const struct pt_config *config)
 {
 	uint32_t last_ctc, ctc, ctc_delta, have_mtc;
 	uint64_t cyc, fc, fcr;
 	int errcode;

 	if (!tcal || !packet || !config)
 		return -pte_internal;

 	last_ctc = tcal->ctc;
 	have_mtc = tcal->have_mtc;
 	cyc = tcal->cyc_mtc;

 	ctc = packet->ctc << config->mtc_freq;

 	/* We need at least two MTC (including this). */
 	if (!have_mtc) {
 		tcal->cyc_mtc = 0ull;
 		tcal->ctc = ctc;
 		tcal->have_mtc = 1;

 		return 0;
 	}

 	/* Without any cycles, we can't calibrate.  Try again at the next
 	 * MTC and distribute the cycles over the combined MTC period.
 	 */
 	if (!cyc)
 		return 0;

 	/* Prepare for the next packet in case we error out below. */
 	tcal->have_mtc = 1;
 	tcal->cyc_mtc = 0ull;
 	tcal->ctc = ctc;

 	/* Let's pretend we will fail.  We'll correct it at the end. */
 	tcal->lost_mtc += 1;

 	errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
 	if (errcode < 0)
 		return errcode;

 	errcode = pt_time_ctc_fc(&fc, ctc_delta, config);
 	if (errcode < 0)
 		return errcode;

 	/* We shift the nominator to improve rounding precision.
 	 *
 	 * Since we're only collecting the CYCs between two MTC, we shouldn't
 	 * overflow.  Let's rather fail than overflow.
 	 */
 	if (fc & ~(~0ull >> pt_tcal_fcr_shr))
 		return -pte_internal;

 	fcr = (fc << pt_tcal_fcr_shr) / cyc;

 	errcode = pt_tcal_set_fcr(tcal, fcr);
 	if (errcode < 0)
 		return errcode;

 	/* We updated the FCR.  This recovers from previous MTC losses. */
 	tcal->lost_mtc = 0;

 	return 0;
 }

 int pt_tcal_update_cyc(struct pt_time_cal *tcal,
 		      const struct pt_packet_cyc *packet,
 		      const struct pt_config *config)
 {
 	uint64_t cyc;

 	(void) config;

 	if (!tcal || !packet)
 		return -pte_internal;

 	cyc = packet->value;
 	tcal->cyc_mtc += cyc;
 	tcal->cyc_tsc += cyc;

 	return 0;
 }
	/*
	* Copyright (c) 2014-2015, Intel Corporation
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions are met:
	*
	* * Redistributions of source code must retain the above copyright notice,
	* this list of conditions and the following disclaimer.
	* * Redistributions in binary form must reproduce the above copyright notice,
	* this list of conditions and the following disclaimer in the documentation
	* and/or other materials provided with the distribution.
	* * Neither the name of Intel Corporation nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
	* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
	* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
	* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	* POSSIBILITY OF SUCH DAMAGE.
	*/

	#include "pt_time.h"

	#include "intel-pt.h"

	#include <string.h>
	#include <limits.h>


	void pt_time_init(struct pt_time *time)
	{
	if (!time)
	return;

	memset(time, 0, sizeof(*time));
	}

	int pt_time_query_tsc(uint64_t tsc, uint32_t lost_mtc,
	uint32_t lost_cyc, const struct pt_time time)
	{
	if (!tsc \|\| !time)
	return -pte_internal;

	*tsc = time->tsc;

	if (lost_mtc)
	*lost_mtc = time->lost_mtc;
	if (lost_cyc)
	*lost_cyc = time->lost_cyc;

	if (!time->have_tsc)
	return -pte_no_time;

	return 0;
	}

	int pt_time_query_cbr(uint32_t cbr, const struct pt_time time)
	{
	if (!cbr \|\| !time)
	return -pte_internal;

	if (!time->have_cbr)
	return -pte_no_cbr;

	*cbr = time->cbr;

	return 0;
	}

	/* Compute the distance between two CTC sources.
	*
	* We adjust a single wrap-around but fail if the distance is bigger than that.
	*
	* Returns zero on success, a negative error code otherwise.
	*/
	static int pt_time_ctc_delta(uint32_t *ctc_delta, uint32_t ctc,
	uint32_t last_ctc, const struct pt_config *config)
	{
	if (!config \|\| !ctc_delta)
	return -pte_internal;

	/* Correct a single wrap-around. If we lost enough MTCs to wrap
	* around twice, timing will be wrong until the next TSC.
	*/
	if (ctc < last_ctc) {
	ctc += 1u << (config->mtc_freq + pt_pl_mtc_bit_size);

	/* Since we only store the CTC between TMA/MTC or MTC/TMC a
	* single correction should suffice.
	*/
	if (ctc < last_ctc)
	return -pte_bad_packet;
	}

	*ctc_delta = ctc - last_ctc;
	return 0;
	}

	/* Translate CTC into the same unit as the FastCounter by multiplying with P.
	*
	* Returns zero on success, a negative error code otherwise.
	*/
	static int pt_time_ctc_fc(uint64_t *fc, uint64_t ctc,
	const struct pt_config *config)
	{
	uint32_t eax, ebx;

	if (!fc \|\| !config)
	return -pte_internal;

	eax = config->cpuid_0x15_eax;
	ebx = config->cpuid_0x15_ebx;

	/* Neither multiply nor divide by zero. */
	if (!eax \|\| !ebx)
	return -pte_bad_config;

	fc = (ctc ebx) / eax;
	return 0;
	}

	int pt_time_update_tsc(struct pt_time *time,
	const struct pt_packet_tsc *packet,
	const struct pt_config *config)
	{
	(void) config;

	if (!time \|\| !packet)
	return -pte_internal;

	time->have_tsc = 1;
	time->have_tma = 0;
	time->tsc = time->base = packet->tsc;
	time->fc = 0ull;

	/* We got the full time; we recover from previous losses. */
	time->lost_mtc = 0;
	time->lost_cyc = 0;

	return 0;
	}

	int pt_time_update_cbr(struct pt_time *time,
	const struct pt_packet_cbr *packet,
	const struct pt_config *config)
	{
	(void) config;

	if (!time \|\| !packet)
	return -pte_internal;

	time->have_cbr = 1;
	time->cbr = packet->ratio;

	return 0;
	}

	int pt_time_update_tma(struct pt_time *time,
	const struct pt_packet_tma *packet,
	const struct pt_config *config)
	{
	uint32_t ctc, mtc_mask, mtc_offset, mtc_freq;
	uint64_t fc;

	if (!time \|\| !packet \|\| !config)
	return -pte_internal;

	/* Without a TSC something is seriously wrong. */
	if (!time->have_tsc)
	return -pte_bad_context;

	ctc = packet->ctc;
	fc = packet->fc;

	mtc_freq = config->mtc_freq;
	mtc_mask = (1u << mtc_freq) - 1u;
	mtc_offset = ctc & mtc_mask;

	/* Mask out the MTC offset and the high order bits. */
	ctc &= pt_pl_mtc_mask << mtc_freq;

	time->have_tma = 1;
	time->base -= fc;
	time->fc += fc;
	time->mtc_offset = mtc_offset;

	/* If the MTC frequency is low enough that TMA provides the full CTC
	* value, we're fine.
	*/
	if ((mtc_freq + 8) <= pt_pl_tma_ctc_bit_size) {
	time->ctc = ctc;
	time->ctc_cyc = ctc;

	/* We can use the TMA instead of an MTC. */
	time->have_mtc = 1;
	}
	/* Otherwise, we must rely on either having seen an MTC before or
	* on heuristically approximating it later at the next MTC.
	*/

	return 0;
	}

	int pt_time_update_mtc(struct pt_time *time,
	const struct pt_packet_mtc *packet,
	const struct pt_config *config)
	{
	uint32_t last_ctc, mtc_offset, ctc, ctc_delta;
	uint64_t tsc, base;
	uint8_t mtc_freq;
	int errcode, have_tsc, have_tma, have_mtc;

	if (!time \|\| !packet \|\| !config)
	return -pte_internal;

	have_tsc = time->have_tsc;
	have_tma = time->have_tma;
	have_mtc = time->have_mtc;

	/* We ignore MTCs between TSC and TMA to avoid apparent CTC overflows.
	* Later MTCs will ensure that no time is lost.
	*/
	if (have_tsc && !have_tma)
	return 0;

	base = time->base;
	last_ctc = time->ctc;
	mtc_offset = time->mtc_offset;
	mtc_freq = config->mtc_freq;

	ctc = packet->ctc << mtc_freq;

	/* Store our CTC value if we have or would have reset FC. */
	if (time->fc \|\| time->lost_cyc \|\| !have_mtc)
	time->ctc_cyc = ctc;

	/* Prepare for the next packet in case we error out below. */
	time->have_mtc = 1;
	time->fc = 0ull;
	time->mtc_offset = 0;
	time->ctc = ctc;

	/* We recover from previous CYC losses. */
	time->lost_cyc = 0;

	/* Avoid a big jump when we see the first MTC with an arbitrary CTC
	* payload.
	*/
	if (!have_mtc) {
	uint8_t shift;

	/* If we have not seen a TMA, we ignore this first MTC.
	*
	* We have no idea where in this MTC period tracing started.
	* We could lose an entire MTC period or just a tiny fraction.
	*
	* On the other hand, if we assumed a previous MTC value, we
	* might make just the same error.
	*/
	if (!have_tma)
	return 0;

	/* The TMA we've seen provided an offset into the current
	* MTC period. Let's assume the last CTC was just small
	* enough to contain that offset.
	*/
	shift = pt_pl_tma_ctc_bit_size;
	if (shift < mtc_freq)
	shift = mtc_freq;

	ctc_delta = 1u << shift;
	} else {
	/* This is the normal case. We have seen an MTC before so we
	* know the previous CTC value.
	*/

	errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
	if (errcode < 0) {
	time->lost_mtc += 1;
	return errcode;
	}
	}

	/* We don't want a wrap-around here. Something must be wrong. */
	if (ctc_delta < mtc_offset) {
	time->lost_mtc += 1;
	return -pte_bad_packet;
	}

	ctc_delta -= mtc_offset;

	errcode = pt_time_ctc_fc(&tsc, ctc_delta, config);
	if (errcode < 0)
	return errcode;

	base += tsc;
	time->tsc = time->base = base;

	return 0;
	}

	/* Adjust a CYC packet's payload spanning multiple MTC periods.
	*
	* CYC packets measure the Fast Counter since the last CYC(-eligible) packet.
	* Depending on the CYC threshold, we may not get a CYC for each MTC, so a CYC
	* period may overlap with or even span multiple MTC periods.
	*
	* We can't do much about the overlap case without examining all packets in
	* the respective periods. We leave this as expected imprecision.
	*
	* If we find a CYC packet to span multiple MTC packets, though, we try to
	* approximate the portion for the current MTC period by subtracting the
	* estimated portion for previous MTC periods using calibration information.
	*
	* We only consider MTC. For the first CYC after TSC, the corresponding TMA
	* will contain the Fast Counter at TSC.
	*
	* Returns zero on success, a negative error code otherwise.
	*/
	static int pt_time_adjust_cyc(uint64_t cyc, const struct pt_time time,
	const struct pt_config *config, uint64_t fcr)
	{
	uint32_t last_ctc, ctc, ctc_delta;
	uint64_t fc, total_cyc, old_cyc;
	int errcode;

	if (!time \|\| !config \|\| !fcr)
	return -pte_internal;

	last_ctc = time->ctc_cyc;
	ctc = time->ctc;

	/* There is nothing to do if this is the current MTC period. */
	if (ctc == last_ctc)
	return 0;

	/* Calibration computes
	*
	* fc = (ctc_delta * cpuid[0x15].ebx) / cpuid[0x15].eax.
	* fcr = (fc << pt_tcal_fcr_shr) / cyc
	*
	* So cyc = (fc << pt_tcal_fcr_shr) / fcr.
	*/

	errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
	if (errcode < 0)
	return errcode;

	errcode = pt_time_ctc_fc(&fc, ctc_delta, config);
	if (errcode < 0)
	return errcode;

	old_cyc = (fc << pt_tcal_fcr_shr) / fcr;
	total_cyc = *cyc;

	/* Make sure we don't wrap around. If we would, attribute the entire
	* CYC payload to any previous MTC period.
	*
	* We lost an unknown portion of the CYC payload for the current MTC
	* period, but it's usually better to run too slow than too fast.
	*/
	if (total_cyc < old_cyc)
	total_cyc = old_cyc;

	*cyc = total_cyc - old_cyc;
	return 0;
	}

	int pt_time_update_cyc(struct pt_time *time,
	const struct pt_packet_cyc *packet,
	const struct pt_config *config, uint64_t fcr)
	{
	uint64_t cyc, fc;

	if (!time \|\| !packet \|\| !config)
	return -pte_internal;

	if (!fcr) {
	time->lost_cyc += 1;
	return 0;
	}

	cyc = packet->value;
	fc = time->fc;
	if (!fc) {
	int errcode;

	errcode = pt_time_adjust_cyc(&cyc, time, config, fcr);
	if (errcode < 0)
	return errcode;
	}

	fc += (cyc * fcr) >> pt_tcal_fcr_shr;

	time->fc = fc;
	time->tsc = time->base + fc;

	return 0;
	}

	void pt_tcal_init(struct pt_time_cal *tcal)
	{
	if (!tcal)
	return;

	memset(tcal, 0, sizeof(*tcal));

	tcal->min_fcr = UINT64_MAX;
	}

	static int pt_tcal_have_fcr(const struct pt_time_cal *tcal)
	{
	if (!tcal)
	return 0;

	return (tcal->min_fcr <= tcal->max_fcr);
	}

	int pt_tcal_fcr(uint64_t fcr, const struct pt_time_cal tcal)
	{
	if (!fcr \|\| !tcal)
	return -pte_internal;

	if (!pt_tcal_have_fcr(tcal))
	return -pte_no_time;

	*fcr = tcal->fcr;

	return 0;
	}

	int pt_tcal_set_fcr(struct pt_time_cal *tcal, uint64_t fcr)
	{
	if (!tcal)
	return -pte_internal;

	tcal->fcr = fcr;

	if (fcr < tcal->min_fcr)
	tcal->min_fcr = fcr;

	if (fcr > tcal->max_fcr)
	tcal->max_fcr = fcr;

	return 0;
	}

	int pt_tcal_update_tsc(struct pt_time_cal *tcal,
	const struct pt_packet_tsc *packet,
	const struct pt_config *config)
	{
	(void) config;

	if (!tcal \|\| !packet)
	return -pte_internal;

	/* A TSC outside of PSB+ may indicate loss of time. We do not use it
	* for calibration. We store the TSC value for calibration at the next
	* TSC in PSB+, though.
	*/
	tcal->tsc = packet->tsc;
	tcal->cyc_tsc = 0ull;

	return 0;
	}

	int pt_tcal_header_tsc(struct pt_time_cal *tcal,
	const struct pt_packet_tsc *packet,
	const struct pt_config *config)
	{
	uint64_t tsc, last_tsc, tsc_delta, cyc, fcr;

	(void) config;

	if (!tcal \|\| !packet)
	return -pte_internal;

	last_tsc = tcal->tsc;
	cyc = tcal->cyc_tsc;

	tsc = packet->tsc;

	tcal->tsc = tsc;
	tcal->cyc_tsc = 0ull;

	if (!last_tsc \|\| !cyc)
	return 0;

	/* Correct a single wrap-around. */
	if (tsc < last_tsc) {
	tsc += 1ull << pt_pl_tsc_bit_size;

	if (tsc < last_tsc)
	return -pte_bad_packet;
	}

	tsc_delta = tsc - last_tsc;

	/* We shift the nominator to improve rounding precision.
	*
	* Since we're only collecting the CYCs between two TSC, we shouldn't
	* overflow. Let's rather fail than overflow.
	*/
	if (tsc_delta & ~(~0ull >> pt_tcal_fcr_shr))
	return -pte_internal;

	fcr = (tsc_delta << pt_tcal_fcr_shr) / cyc;

	return pt_tcal_set_fcr(tcal, fcr);
	}

	int pt_tcal_update_cbr(struct pt_time_cal *tcal,
	const struct pt_packet_cbr *packet,
	const struct pt_config *config)
	{
	/* A CBR outside of PSB+ indicates a frequency change. Reset our
	* calibration state.
	*/
	pt_tcal_init(tcal);

	return pt_tcal_header_cbr(tcal, packet, config);
	}

	int pt_tcal_header_cbr(struct pt_time_cal *tcal,
	const struct pt_packet_cbr *packet,
	const struct pt_config *config)
	{
	uint64_t cbr, p1, fcr;

	if (!tcal \|\| !packet \|\| !config)
	return -pte_internal;

	p1 = config->nom_freq;
	if (!p1)
	return 0;

	/* If we know the nominal frequency, we can use it for calibration. */
	cbr = packet->ratio;

	fcr = (p1 << pt_tcal_fcr_shr) / cbr;

	return pt_tcal_set_fcr(tcal, fcr);
	}

	int pt_tcal_update_tma(struct pt_time_cal *tcal,
	const struct pt_packet_tma *packet,
	const struct pt_config *config)
	{
	(void) tcal;
	(void) packet;
	(void) config;

	/* Nothing to do. */
	return 0;
	}

	int pt_tcal_update_mtc(struct pt_time_cal *tcal,
	const struct pt_packet_mtc *packet,
	const struct pt_config *config)
	{
	uint32_t last_ctc, ctc, ctc_delta, have_mtc;
	uint64_t cyc, fc, fcr;
	int errcode;

	if (!tcal \|\| !packet \|\| !config)
	return -pte_internal;

	last_ctc = tcal->ctc;
	have_mtc = tcal->have_mtc;
	cyc = tcal->cyc_mtc;

	ctc = packet->ctc << config->mtc_freq;

	/* We need at least two MTC (including this). */
	if (!have_mtc) {
	tcal->cyc_mtc = 0ull;
	tcal->ctc = ctc;
	tcal->have_mtc = 1;

	return 0;
	}

	/* Without any cycles, we can't calibrate. Try again at the next
	* MTC and distribute the cycles over the combined MTC period.
	*/
	if (!cyc)
	return 0;

	/* Prepare for the next packet in case we error out below. */
	tcal->have_mtc = 1;
	tcal->cyc_mtc = 0ull;
	tcal->ctc = ctc;

	/* Let's pretend we will fail. We'll correct it at the end. */
	tcal->lost_mtc += 1;

	errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
	if (errcode < 0)
	return errcode;

	errcode = pt_time_ctc_fc(&fc, ctc_delta, config);
	if (errcode < 0)
	return errcode;

	/* We shift the nominator to improve rounding precision.
	*
	* Since we're only collecting the CYCs between two MTC, we shouldn't
	* overflow. Let's rather fail than overflow.
	*/
	if (fc & ~(~0ull >> pt_tcal_fcr_shr))
	return -pte_internal;

	fcr = (fc << pt_tcal_fcr_shr) / cyc;

	errcode = pt_tcal_set_fcr(tcal, fcr);
	if (errcode < 0)
	return errcode;

	/* We updated the FCR. This recovers from previous MTC losses. */
	tcal->lost_mtc = 0;

	return 0;
	}

	int pt_tcal_update_cyc(struct pt_time_cal *tcal,
	const struct pt_packet_cyc *packet,
	const struct pt_config *config)
	{
	uint64_t cyc;

	(void) config;

	if (!tcal \|\| !packet)
	return -pte_internal;

	cyc = packet->value;
	tcal->cyc_mtc += cyc;
	tcal->cyc_tsc += cyc;

	return 0;
	}