tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 /*
   2  * intel_pt_decoder.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #ifndef _GNU_SOURCE
  17 #define _GNU_SOURCE
  18 #endif
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdint.h>
  24 #include <inttypes.h>
  25 #include <linux/compiler.h>
  26
  27 #include "../cache.h"
  28 #include "../util.h"
  29
  30 #include "intel-pt-insn-decoder.h"
  31 #include "intel-pt-pkt-decoder.h"
  32 #include "intel-pt-decoder.h"
  33 #include "intel-pt-log.h"
  34
  35 #define INTEL_PT_BLK_SIZE 1024
  36
  37 #define BIT63 (((uint64_t)1 << 63))
  38
  39 #define INTEL_PT_RETURN 1
  40
  41 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  42 #define INTEL_PT_MAX_LOOPS 10000
  43
  44 struct intel_pt_blk {
  45         struct intel_pt_blk *prev;
  46         uint64_t ip[INTEL_PT_BLK_SIZE];
  47 };
  48
  49 struct intel_pt_stack {
  50         struct intel_pt_blk *blk;
  51         struct intel_pt_blk *spare;
  52         int pos;
  53 };
  54
  55 enum intel_pt_pkt_state {
  56         INTEL_PT_STATE_NO_PSB,
  57         INTEL_PT_STATE_NO_IP,
  58         INTEL_PT_STATE_ERR_RESYNC,
  59         INTEL_PT_STATE_IN_SYNC,
  60         INTEL_PT_STATE_TNT,
  61         INTEL_PT_STATE_TIP,
  62         INTEL_PT_STATE_TIP_PGD,
  63         INTEL_PT_STATE_FUP,
  64         INTEL_PT_STATE_FUP_NO_TIP,
  65 };
  66
  67 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  68 {
  69         switch (pkt_state) {
  70         case INTEL_PT_STATE_NO_PSB:
  71         case INTEL_PT_STATE_NO_IP:
  72         case INTEL_PT_STATE_ERR_RESYNC:
  73         case INTEL_PT_STATE_IN_SYNC:
  74         case INTEL_PT_STATE_TNT:
  75                 return true;
  76         case INTEL_PT_STATE_TIP:
  77         case INTEL_PT_STATE_TIP_PGD:
  78         case INTEL_PT_STATE_FUP:
  79         case INTEL_PT_STATE_FUP_NO_TIP:
  80                 return false;
  81         default:
  82                 return true;
  83         };
  84 }
  85
  86 #ifdef INTEL_PT_STRICT
  87 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
  88 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
  89 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
  90 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
  91 #else
  92 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
  93 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
  94 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
  95 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
  96 #endif
  97
  98 struct intel_pt_decoder {
  99         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 100         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 101                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 102                          uint64_t max_insn_cnt, void *data);
 103         void *data;
 104         struct intel_pt_state state;
 105         const unsigned char *buf;
 106         size_t len;
 107         bool return_compression;
 108         bool mtc_insn;
 109         bool pge;
 110         bool have_tma;
 111         bool have_cyc;
 112         bool fixup_last_mtc;
 113         bool have_last_ip;
 114         enum intel_pt_param_flags flags;
 115         uint64_t pos;
 116         uint64_t last_ip;
 117         uint64_t ip;
 118         uint64_t cr3;
 119         uint64_t timestamp;
 120         uint64_t tsc_timestamp;
 121         uint64_t ref_timestamp;
 122         uint64_t sample_timestamp;
 123         uint64_t ret_addr;
 124         uint64_t ctc_timestamp;
 125         uint64_t ctc_delta;
 126         uint64_t cycle_cnt;
 127         uint64_t cyc_ref_timestamp;
 128         uint32_t last_mtc;
 129         uint32_t tsc_ctc_ratio_n;
 130         uint32_t tsc_ctc_ratio_d;
 131         uint32_t tsc_ctc_mult;
 132         uint32_t tsc_slip;
 133         uint32_t ctc_rem_mask;
 134         int mtc_shift;
 135         struct intel_pt_stack stack;
 136         enum intel_pt_pkt_state pkt_state;
 137         struct intel_pt_pkt packet;
 138         struct intel_pt_pkt tnt;
 139         int pkt_step;
 140         int pkt_len;
 141         int last_packet_type;
 142         unsigned int cbr;
 143         unsigned int max_non_turbo_ratio;
 144         double max_non_turbo_ratio_fp;
 145         double cbr_cyc_to_tsc;
 146         double calc_cyc_to_tsc;
 147         bool have_calc_cyc_to_tsc;
 148         int exec_mode;
 149         unsigned int insn_bytes;
 150         uint64_t period;
 151         enum intel_pt_period_type period_type;
 152         uint64_t tot_insn_cnt;
 153         uint64_t period_insn_cnt;
 154         uint64_t period_mask;
 155         uint64_t period_ticks;
 156         uint64_t last_masked_timestamp;
 157         bool continuous_period;
 158         bool overflow;
 159         bool set_fup_tx_flags;
 160         unsigned int fup_tx_flags;
 161         unsigned int tx_flags;
 162         uint64_t timestamp_insn_cnt;
 163         uint64_t sample_insn_cnt;
 164         uint64_t stuck_ip;
 165         int no_progress;
 166         int stuck_ip_prd;
 167         int stuck_ip_cnt;
 168         const unsigned char *next_buf;
 169         size_t next_len;
 170         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 171 };
 172
 173 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 174 {
 175         int i;
 176
 177         for (i = 0; x != 1; i++)
 178                 x >>= 1;
 179
 180         return x << i;
 181 }
 182
 183 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 184 {
 185         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 186                 uint64_t period;
 187
 188                 period = intel_pt_lower_power_of_2(decoder->period);
 189                 decoder->period_mask  = ~(period - 1);
 190                 decoder->period_ticks = period;
 191         }
 192 }
 193
 194 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 195 {
 196         if (!d)
 197                 return 0;
 198         return (t / d) * n + ((t % d) * n) / d;
 199 }
 200
 201 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 202 {
 203         struct intel_pt_decoder *decoder;
 204
 205         if (!params->get_trace || !params->walk_insn)
 206                 return NULL;
 207
 208         decoder = zalloc(sizeof(struct intel_pt_decoder));
 209         if (!decoder)
 210                 return NULL;
 211
 212         decoder->get_trace          = params->get_trace;
 213         decoder->walk_insn          = params->walk_insn;
 214         decoder->data               = params->data;
 215         decoder->return_compression = params->return_compression;
 216
 217         decoder->flags              = params->flags;
 218
 219         decoder->period             = params->period;
 220         decoder->period_type        = params->period_type;
 221
 222         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 223         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 224
 225         intel_pt_setup_period(decoder);
 226
 227         decoder->mtc_shift = params->mtc_period;
 228         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 229
 230         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 231         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 232
 233         if (!decoder->tsc_ctc_ratio_n)
 234                 decoder->tsc_ctc_ratio_d = 0;
 235
 236         if (decoder->tsc_ctc_ratio_d) {
 237                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 238                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 239                                                 decoder->tsc_ctc_ratio_d;
 240
 241                 /*
 242                  * Allow for timestamps appearing to backwards because a TSC
 243                  * packet has slipped past a MTC packet, so allow 2 MTC ticks
 244                  * or ...
 245                  */
 246                 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,
 247                                         decoder->tsc_ctc_ratio_n,
 248                                         decoder->tsc_ctc_ratio_d);
 249         }
 250         /* ... or 0x100 paranoia */
 251         if (decoder->tsc_slip < 0x100)
 252                 decoder->tsc_slip = 0x100;
 253
 254         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 255         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 256         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 257         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 258         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 259
 260         return decoder;
 261 }
 262
 263 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 264 {
 265         struct intel_pt_blk *blk = stack->blk;
 266
 267         stack->blk = blk->prev;
 268         if (!stack->spare)
 269                 stack->spare = blk;
 270         else
 271                 free(blk);
 272 }
 273
 274 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 275 {
 276         if (!stack->pos) {
 277                 if (!stack->blk)
 278                         return 0;
 279                 intel_pt_pop_blk(stack);
 280                 if (!stack->blk)
 281                         return 0;
 282                 stack->pos = INTEL_PT_BLK_SIZE;
 283         }
 284         return stack->blk->ip[--stack->pos];
 285 }
 286
 287 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 288 {
 289         struct intel_pt_blk *blk;
 290
 291         if (stack->spare) {
 292                 blk = stack->spare;
 293                 stack->spare = NULL;
 294         } else {
 295                 blk = malloc(sizeof(struct intel_pt_blk));
 296                 if (!blk)
 297                         return -ENOMEM;
 298         }
 299
 300         blk->prev = stack->blk;
 301         stack->blk = blk;
 302         stack->pos = 0;
 303         return 0;
 304 }
 305
 306 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 307 {
 308         int err;
 309
 310         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 311                 err = intel_pt_alloc_blk(stack);
 312                 if (err)
 313                         return err;
 314         }
 315
 316         stack->blk->ip[stack->pos++] = ip;
 317         return 0;
 318 }
 319
 320 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 321 {
 322         while (stack->blk)
 323                 intel_pt_pop_blk(stack);
 324         stack->pos = 0;
 325 }
 326
 327 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 328 {
 329         intel_pt_clear_stack(stack);
 330         zfree(&stack->blk);
 331         zfree(&stack->spare);
 332 }
 333
 334 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 335 {
 336         intel_pt_free_stack(&decoder->stack);
 337         free(decoder);
 338 }
 339
 340 static int intel_pt_ext_err(int code)
 341 {
 342         switch (code) {
 343         case -ENOMEM:
 344                 return INTEL_PT_ERR_NOMEM;
 345         case -ENOSYS:
 346                 return INTEL_PT_ERR_INTERN;
 347         case -EBADMSG:
 348                 return INTEL_PT_ERR_BADPKT;
 349         case -ENODATA:
 350                 return INTEL_PT_ERR_NODATA;
 351         case -EILSEQ:
 352                 return INTEL_PT_ERR_NOINSN;
 353         case -ENOENT:
 354                 return INTEL_PT_ERR_MISMAT;
 355         case -EOVERFLOW:
 356                 return INTEL_PT_ERR_OVR;
 357         case -ENOSPC:
 358                 return INTEL_PT_ERR_LOST;
 359         case -ELOOP:
 360                 return INTEL_PT_ERR_NELOOP;
 361         default:
 362                 return INTEL_PT_ERR_UNK;
 363         }
 364 }
 365
 366 static const char *intel_pt_err_msgs[] = {
 367         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 368         [INTEL_PT_ERR_INTERN] = "Internal error",
 369         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 370         [INTEL_PT_ERR_NODATA] = "No more data",
 371         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 372         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 373         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 374         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 375         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 376         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 377 };
 378
 379 int intel_pt__strerror(int code, char *buf, size_t buflen)
 380 {
 381         if (code < 1 || code > INTEL_PT_ERR_MAX)
 382                 code = INTEL_PT_ERR_UNK;
 383         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 384         return 0;
 385 }
 386
 387 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 388                                  uint64_t last_ip)
 389 {
 390         uint64_t ip;
 391
 392         switch (packet->count) {
 393         case 1:
 394                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 395                      packet->payload;
 396                 break;
 397         case 2:
 398                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 399                      packet->payload;
 400                 break;
 401         case 3:
 402                 ip = packet->payload;
 403                 /* Sign-extend 6-byte ip */
 404                 if (ip & (uint64_t)0x800000000000ULL)
 405                         ip |= (uint64_t)0xffff000000000000ULL;
 406                 break;
 407         case 4:
 408                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 409                      packet->payload;
 410                 break;
 411         case 6:
 412                 ip = packet->payload;
 413                 break;
 414         default:
 415                 return 0;
 416         }
 417
 418         return ip;
 419 }
 420
 421 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 422 {
 423         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 424         decoder->have_last_ip = true;
 425 }
 426
 427 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 428 {
 429         intel_pt_set_last_ip(decoder);
 430         decoder->ip = decoder->last_ip;
 431 }
 432
 433 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 434 {
 435         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 436                             decoder->buf);
 437 }
 438
 439 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 440 {
 441         intel_pt_log("ERROR: Internal error\n");
 442         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 443         return -ENOSYS;
 444 }
 445
 446 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 447 {
 448         decoder->tx_flags = 0;
 449 }
 450
 451 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 452 {
 453         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 454 }
 455
 456 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 457 {
 458         intel_pt_clear_tx_flags(decoder);
 459         decoder->have_tma = false;
 460         decoder->pkt_len = 1;
 461         decoder->pkt_step = 1;
 462         intel_pt_decoder_log_packet(decoder);
 463         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 464                 intel_pt_log("ERROR: Bad packet\n");
 465                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 466         }
 467         return -EBADMSG;
 468 }
 469
 470 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 471 {
 472         struct intel_pt_buffer buffer = { .buf = 0, };
 473         int ret;
 474
 475         decoder->pkt_step = 0;
 476
 477         intel_pt_log("Getting more data\n");
 478         ret = decoder->get_trace(&buffer, decoder->data);
 479         if (ret)
 480                 return ret;
 481         decoder->buf = buffer.buf;
 482         decoder->len = buffer.len;
 483         if (!decoder->len) {
 484                 intel_pt_log("No more data\n");
 485                 return -ENODATA;
 486         }
 487         if (!buffer.consecutive) {
 488                 decoder->ip = 0;
 489                 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 490                 decoder->ref_timestamp = buffer.ref_timestamp;
 491                 decoder->timestamp = 0;
 492                 decoder->have_tma = false;
 493                 decoder->state.trace_nr = buffer.trace_nr;
 494                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 495                              decoder->ref_timestamp);
 496                 return -ENOLINK;
 497         }
 498
 499         return 0;
 500 }
 501
 502 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 503 {
 504         if (!decoder->next_buf)
 505                 return intel_pt_get_data(decoder);
 506
 507         decoder->buf = decoder->next_buf;
 508         decoder->len = decoder->next_len;
 509         decoder->next_buf = 0;
 510         decoder->next_len = 0;
 511         return 0;
 512 }
 513
 514 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 515 {
 516         unsigned char *buf = decoder->temp_buf;
 517         size_t old_len, len, n;
 518         int ret;
 519
 520         old_len = decoder->len;
 521         len = decoder->len;
 522         memcpy(buf, decoder->buf, len);
 523
 524         ret = intel_pt_get_data(decoder);
 525         if (ret) {
 526                 decoder->pos += old_len;
 527                 return ret < 0 ? ret : -EINVAL;
 528         }
 529
 530         n = INTEL_PT_PKT_MAX_SZ - len;
 531         if (n > decoder->len)
 532                 n = decoder->len;
 533         memcpy(buf + len, decoder->buf, n);
 534         len += n;
 535
 536         ret = intel_pt_get_packet(buf, len, &decoder->packet);
 537         if (ret < (int)old_len) {
 538                 decoder->next_buf = decoder->buf;
 539                 decoder->next_len = decoder->len;
 540                 decoder->buf = buf;
 541                 decoder->len = old_len;
 542                 return intel_pt_bad_packet(decoder);
 543         }
 544
 545         decoder->next_buf = decoder->buf + (ret - old_len);
 546         decoder->next_len = decoder->len - (ret - old_len);
 547
 548         decoder->buf = buf;
 549         decoder->len = ret;
 550
 551         return ret;
 552 }
 553
 554 struct intel_pt_pkt_info {
 555         struct intel_pt_decoder   *decoder;
 556         struct intel_pt_pkt       packet;
 557         uint64_t                  pos;
 558         int                       pkt_len;
 559         int                       last_packet_type;
 560         void                      *data;
 561 };
 562
 563 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 564
 565 /* Lookahead packets in current buffer */
 566 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 567                                   intel_pt_pkt_cb_t cb, void *data)
 568 {
 569         struct intel_pt_pkt_info pkt_info;
 570         const unsigned char *buf = decoder->buf;
 571         size_t len = decoder->len;
 572         int ret;
 573
 574         pkt_info.decoder          = decoder;
 575         pkt_info.pos              = decoder->pos;
 576         pkt_info.pkt_len          = decoder->pkt_step;
 577         pkt_info.last_packet_type = decoder->last_packet_type;
 578         pkt_info.data             = data;
 579
 580         while (1) {
 581                 do {
 582                         pkt_info.pos += pkt_info.pkt_len;
 583                         buf          += pkt_info.pkt_len;
 584                         len          -= pkt_info.pkt_len;
 585
 586                         if (!len)
 587                                 return INTEL_PT_NEED_MORE_BYTES;
 588
 589                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 590                         if (!ret)
 591                                 return INTEL_PT_NEED_MORE_BYTES;
 592                         if (ret < 0)
 593                                 return ret;
 594
 595                         pkt_info.pkt_len = ret;
 596                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 597
 598                 ret = cb(&pkt_info);
 599                 if (ret)
 600                         return 0;
 601
 602                 pkt_info.last_packet_type = pkt_info.packet.type;
 603         }
 604 }
 605
 606 struct intel_pt_calc_cyc_to_tsc_info {
 607         uint64_t        cycle_cnt;
 608         unsigned int    cbr;
 609         uint32_t        last_mtc;
 610         uint64_t        ctc_timestamp;
 611         uint64_t        ctc_delta;
 612         uint64_t        tsc_timestamp;
 613         uint64_t        timestamp;
 614         bool            have_tma;
 615         bool            fixup_last_mtc;
 616         bool            from_mtc;
 617         double          cbr_cyc_to_tsc;
 618 };
 619
 620 /*
 621  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 622  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 623  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 624  * packet by copying the missing bits from the current MTC assuming the least
 625  * difference between the two, and that the current MTC comes after last_mtc.
 626  */
 627 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 628                                     uint32_t *last_mtc)
 629 {
 630         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 631         uint32_t mask = ~(first_missing_bit - 1);
 632
 633         *last_mtc |= mtc & mask;
 634         if (*last_mtc >= mtc) {
 635                 *last_mtc -= first_missing_bit;
 636                 *last_mtc &= 0xff;
 637         }
 638 }
 639
 640 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 641 {
 642         struct intel_pt_decoder *decoder = pkt_info->decoder;
 643         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 644         uint64_t timestamp;
 645         double cyc_to_tsc;
 646         unsigned int cbr;
 647         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 648
 649         switch (pkt_info->packet.type) {
 650         case INTEL_PT_TNT:
 651         case INTEL_PT_TIP_PGE:
 652         case INTEL_PT_TIP:
 653         case INTEL_PT_FUP:
 654         case INTEL_PT_PSB:
 655         case INTEL_PT_PIP:
 656         case INTEL_PT_MODE_EXEC:
 657         case INTEL_PT_MODE_TSX:
 658         case INTEL_PT_PSBEND:
 659         case INTEL_PT_PAD:
 660         case INTEL_PT_VMCS:
 661         case INTEL_PT_MNT:
 662                 return 0;
 663
 664         case INTEL_PT_MTC:
 665                 if (!data->have_tma)
 666                         return 0;
 667
 668                 mtc = pkt_info->packet.payload;
 669                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 670                         data->fixup_last_mtc = false;
 671                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 672                                                 &data->last_mtc);
 673                 }
 674                 if (mtc > data->last_mtc)
 675                         mtc_delta = mtc - data->last_mtc;
 676                 else
 677                         mtc_delta = mtc + 256 - data->last_mtc;
 678                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 679                 data->last_mtc = mtc;
 680
 681                 if (decoder->tsc_ctc_mult) {
 682                         timestamp = data->ctc_timestamp +
 683                                 data->ctc_delta * decoder->tsc_ctc_mult;
 684                 } else {
 685                         timestamp = data->ctc_timestamp +
 686                                 multdiv(data->ctc_delta,
 687                                         decoder->tsc_ctc_ratio_n,
 688                                         decoder->tsc_ctc_ratio_d);
 689                 }
 690
 691                 if (timestamp < data->timestamp)
 692                         return 1;
 693
 694                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 695                         data->timestamp = timestamp;
 696                         return 0;
 697                 }
 698
 699                 break;
 700
 701         case INTEL_PT_TSC:
 702                 timestamp = pkt_info->packet.payload |
 703                             (data->timestamp & (0xffULL << 56));
 704                 if (data->from_mtc && timestamp < data->timestamp &&
 705                     data->timestamp - timestamp < decoder->tsc_slip)
 706                         return 1;
 707                 if (timestamp < data->timestamp)
 708                         timestamp += (1ULL << 56);
 709                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 710                         if (data->from_mtc)
 711                                 return 1;
 712                         data->tsc_timestamp = timestamp;
 713                         data->timestamp = timestamp;
 714                         return 0;
 715                 }
 716                 break;
 717
 718         case INTEL_PT_TMA:
 719                 if (data->from_mtc)
 720                         return 1;
 721
 722                 if (!decoder->tsc_ctc_ratio_d)
 723                         return 0;
 724
 725                 ctc = pkt_info->packet.payload;
 726                 fc = pkt_info->packet.count;
 727                 ctc_rem = ctc & decoder->ctc_rem_mask;
 728
 729                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 730
 731                 data->ctc_timestamp = data->tsc_timestamp - fc;
 732                 if (decoder->tsc_ctc_mult) {
 733                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 734                 } else {
 735                         data->ctc_timestamp -=
 736                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 737                                         decoder->tsc_ctc_ratio_d);
 738                 }
 739
 740                 data->ctc_delta = 0;
 741                 data->have_tma = true;
 742                 data->fixup_last_mtc = true;
 743
 744                 return 0;
 745
 746         case INTEL_PT_CYC:
 747                 data->cycle_cnt += pkt_info->packet.payload;
 748                 return 0;
 749
 750         case INTEL_PT_CBR:
 751                 cbr = pkt_info->packet.payload;
 752                 if (data->cbr && data->cbr != cbr)
 753                         return 1;
 754                 data->cbr = cbr;
 755                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 756                 return 0;
 757
 758         case INTEL_PT_TIP_PGD:
 759         case INTEL_PT_TRACESTOP:
 760         case INTEL_PT_OVF:
 761         case INTEL_PT_BAD: /* Does not happen */
 762         default:
 763                 return 1;
 764         }
 765
 766         if (!data->cbr && decoder->cbr) {
 767                 data->cbr = decoder->cbr;
 768                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 769         }
 770
 771         if (!data->cycle_cnt)
 772                 return 1;
 773
 774         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 775
 776         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 777             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 778                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 779                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 780                 return 1;
 781         }
 782
 783         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 784         decoder->have_calc_cyc_to_tsc = true;
 785
 786         if (data->cbr) {
 787                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 788                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 789         } else {
 790                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 791                              cyc_to_tsc, pkt_info->pos);
 792         }
 793
 794         return 1;
 795 }
 796
 797 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 798                                      bool from_mtc)
 799 {
 800         struct intel_pt_calc_cyc_to_tsc_info data = {
 801                 .cycle_cnt      = 0,
 802                 .cbr            = 0,
 803                 .last_mtc       = decoder->last_mtc,
 804                 .ctc_timestamp  = decoder->ctc_timestamp,
 805                 .ctc_delta      = decoder->ctc_delta,
 806                 .tsc_timestamp  = decoder->tsc_timestamp,
 807                 .timestamp      = decoder->timestamp,
 808                 .have_tma       = decoder->have_tma,
 809                 .fixup_last_mtc = decoder->fixup_last_mtc,
 810                 .from_mtc       = from_mtc,
 811                 .cbr_cyc_to_tsc = 0,
 812         };
 813
 814         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 815 }
 816
 817 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 818 {
 819         int ret;
 820
 821         decoder->last_packet_type = decoder->packet.type;
 822
 823         do {
 824                 decoder->pos += decoder->pkt_step;
 825                 decoder->buf += decoder->pkt_step;
 826                 decoder->len -= decoder->pkt_step;
 827
 828                 if (!decoder->len) {
 829                         ret = intel_pt_get_next_data(decoder);
 830                         if (ret)
 831                                 return ret;
 832                 }
 833
 834                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
 835                                           &decoder->packet);
 836                 if (ret == INTEL_PT_NEED_MORE_BYTES &&
 837                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 838                         ret = intel_pt_get_split_packet(decoder);
 839                         if (ret < 0)
 840                                 return ret;
 841                 }
 842                 if (ret <= 0)
 843                         return intel_pt_bad_packet(decoder);
 844
 845                 decoder->pkt_len = ret;
 846                 decoder->pkt_step = ret;
 847                 intel_pt_decoder_log_packet(decoder);
 848         } while (decoder->packet.type == INTEL_PT_PAD);
 849
 850         return 0;
 851 }
 852
 853 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 854 {
 855         uint64_t timestamp, masked_timestamp;
 856
 857         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 858         masked_timestamp = timestamp & decoder->period_mask;
 859         if (decoder->continuous_period) {
 860                 if (masked_timestamp != decoder->last_masked_timestamp)
 861                         return 1;
 862         } else {
 863                 timestamp += 1;
 864                 masked_timestamp = timestamp & decoder->period_mask;
 865                 if (masked_timestamp != decoder->last_masked_timestamp) {
 866                         decoder->last_masked_timestamp = masked_timestamp;
 867                         decoder->continuous_period = true;
 868                 }
 869         }
 870         return decoder->period_ticks - (timestamp - masked_timestamp);
 871 }
 872
 873 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 874 {
 875         switch (decoder->period_type) {
 876         case INTEL_PT_PERIOD_INSTRUCTIONS:
 877                 return decoder->period - decoder->period_insn_cnt;
 878         case INTEL_PT_PERIOD_TICKS:
 879                 return intel_pt_next_period(decoder);
 880         case INTEL_PT_PERIOD_NONE:
 881         case INTEL_PT_PERIOD_MTC:
 882         default:
 883                 return 0;
 884         }
 885 }
 886
 887 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 888 {
 889         uint64_t timestamp, masked_timestamp;
 890
 891         switch (decoder->period_type) {
 892         case INTEL_PT_PERIOD_INSTRUCTIONS:
 893                 decoder->period_insn_cnt = 0;
 894                 break;
 895         case INTEL_PT_PERIOD_TICKS:
 896                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 897                 masked_timestamp = timestamp & decoder->period_mask;
 898                 decoder->last_masked_timestamp = masked_timestamp;
 899                 break;
 900         case INTEL_PT_PERIOD_NONE:
 901         case INTEL_PT_PERIOD_MTC:
 902         default:
 903                 break;
 904         }
 905
 906         decoder->state.type |= INTEL_PT_INSTRUCTION;
 907 }
 908
 909 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 910                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 911 {
 912         uint64_t max_insn_cnt, insn_cnt = 0;
 913         int err;
 914
 915         if (!decoder->mtc_insn)
 916                 decoder->mtc_insn = true;
 917
 918         max_insn_cnt = intel_pt_next_sample(decoder);
 919
 920         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 921                                  max_insn_cnt, decoder->data);
 922
 923         decoder->tot_insn_cnt += insn_cnt;
 924         decoder->timestamp_insn_cnt += insn_cnt;
 925         decoder->sample_insn_cnt += insn_cnt;
 926         decoder->period_insn_cnt += insn_cnt;
 927
 928         if (err) {
 929                 decoder->no_progress = 0;
 930                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
 931                 intel_pt_log_at("ERROR: Failed to get instruction",
 932                                 decoder->ip);
 933                 if (err == -ENOENT)
 934                         return -ENOLINK;
 935                 return -EILSEQ;
 936         }
 937
 938         if (ip && decoder->ip == ip) {
 939                 err = -EAGAIN;
 940                 goto out;
 941         }
 942
 943         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 944                 intel_pt_sample_insn(decoder);
 945
 946         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 947                 decoder->state.type = INTEL_PT_INSTRUCTION;
 948                 decoder->state.from_ip = decoder->ip;
 949                 decoder->state.to_ip = 0;
 950                 decoder->ip += intel_pt_insn->length;
 951                 err = INTEL_PT_RETURN;
 952                 goto out;
 953         }
 954
 955         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
 956                 /* Zero-length calls are excluded */
 957                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
 958                     intel_pt_insn->rel) {
 959                         err = intel_pt_push(&decoder->stack, decoder->ip +
 960                                             intel_pt_insn->length);
 961                         if (err)
 962                                 goto out;
 963                 }
 964         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
 965                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
 966         }
 967
 968         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
 969                 int cnt = decoder->no_progress++;
 970
 971                 decoder->state.from_ip = decoder->ip;
 972                 decoder->ip += intel_pt_insn->length +
 973                                 intel_pt_insn->rel;
 974                 decoder->state.to_ip = decoder->ip;
 975                 err = INTEL_PT_RETURN;
 976
 977                 /*
 978                  * Check for being stuck in a loop.  This can happen if a
 979                  * decoder error results in the decoder erroneously setting the
 980                  * ip to an address that is itself in an infinite loop that
 981                  * consumes no packets.  When that happens, there must be an
 982                  * unconditional branch.
 983                  */
 984                 if (cnt) {
 985                         if (cnt == 1) {
 986                                 decoder->stuck_ip = decoder->state.to_ip;
 987                                 decoder->stuck_ip_prd = 1;
 988                                 decoder->stuck_ip_cnt = 1;
 989                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
 990                                    decoder->state.to_ip == decoder->stuck_ip) {
 991                                 intel_pt_log_at("ERROR: Never-ending loop",
 992                                                 decoder->state.to_ip);
 993                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
 994                                 err = -ELOOP;
 995                                 goto out;
 996                         } else if (!--decoder->stuck_ip_cnt) {
 997                                 decoder->stuck_ip_prd += 1;
 998                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
 999                                 decoder->stuck_ip = decoder->state.to_ip;
1000                         }
1001                 }
1002                 goto out_no_progress;
1003         }
1004 out:
1005         decoder->no_progress = 0;
1006 out_no_progress:
1007         decoder->state.insn_op = intel_pt_insn->op;
1008         decoder->state.insn_len = intel_pt_insn->length;
1009
1010         if (decoder->tx_flags & INTEL_PT_IN_TX)
1011                 decoder->state.flags |= INTEL_PT_IN_TX;
1012
1013         return err;
1014 }
1015
1016 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1017                                           struct intel_pt_insn *intel_pt_insn,
1018                                           uint64_t ip, int err)
1019 {
1020         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1021                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1022                ip == decoder->ip + intel_pt_insn->length;
1023 }
1024
1025 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1026 {
1027         struct intel_pt_insn intel_pt_insn;
1028         uint64_t ip;
1029         int err;
1030
1031         ip = decoder->last_ip;
1032
1033         while (1) {
1034                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1035                 if (err == INTEL_PT_RETURN)
1036                         return 0;
1037                 if (err == -EAGAIN ||
1038                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1039                         if (decoder->set_fup_tx_flags) {
1040                                 decoder->set_fup_tx_flags = false;
1041                                 decoder->tx_flags = decoder->fup_tx_flags;
1042                                 decoder->state.type = INTEL_PT_TRANSACTION;
1043                                 decoder->state.from_ip = decoder->ip;
1044                                 decoder->state.to_ip = 0;
1045                                 decoder->state.flags = decoder->fup_tx_flags;
1046                                 return 0;
1047                         }
1048                         return -EAGAIN;
1049                 }
1050                 decoder->set_fup_tx_flags = false;
1051                 if (err)
1052                         return err;
1053
1054                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1055                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1056                                         decoder->ip);
1057                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1058                         return -ENOENT;
1059                 }
1060
1061                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1062                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1063                                         decoder->ip);
1064                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1065                         return -ENOENT;
1066                 }
1067
1068                 intel_pt_bug(decoder);
1069         }
1070 }
1071
1072 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1073 {
1074         struct intel_pt_insn intel_pt_insn;
1075         int err;
1076
1077         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1078         if (err == INTEL_PT_RETURN)
1079                 return 0;
1080         if (err)
1081                 return err;
1082
1083         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1084                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1085                         decoder->pge = false;
1086                         decoder->continuous_period = false;
1087                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1088                         decoder->state.from_ip = decoder->ip;
1089                         decoder->state.to_ip = 0;
1090                         if (decoder->packet.count != 0)
1091                                 decoder->ip = decoder->last_ip;
1092                 } else {
1093                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1094                         decoder->state.from_ip = decoder->ip;
1095                         if (decoder->packet.count == 0) {
1096                                 decoder->state.to_ip = 0;
1097                         } else {
1098                                 decoder->state.to_ip = decoder->last_ip;
1099                                 decoder->ip = decoder->last_ip;
1100                         }
1101                 }
1102                 return 0;
1103         }
1104
1105         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1106                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1107                                 decoder->ip);
1108                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1109                 return -ENOENT;
1110         }
1111
1112         return intel_pt_bug(decoder);
1113 }
1114
1115 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1116 {
1117         struct intel_pt_insn intel_pt_insn;
1118         int err;
1119
1120         while (1) {
1121                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1122                 if (err == INTEL_PT_RETURN)
1123                         return 0;
1124                 if (err)
1125                         return err;
1126
1127                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1128                         if (!decoder->return_compression) {
1129                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1130                                                 decoder->ip);
1131                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1132                                 return -ENOENT;
1133                         }
1134                         if (!decoder->ret_addr) {
1135                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1136                                                 decoder->ip);
1137                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1138                                 return -ENOENT;
1139                         }
1140                         if (!(decoder->tnt.payload & BIT63)) {
1141                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1142                                                 decoder->ip);
1143                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1144                                 return -ENOENT;
1145                         }
1146                         decoder->tnt.count -= 1;
1147                         if (!decoder->tnt.count)
1148                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1149                         decoder->tnt.payload <<= 1;
1150                         decoder->state.from_ip = decoder->ip;
1151                         decoder->ip = decoder->ret_addr;
1152                         decoder->state.to_ip = decoder->ip;
1153                         return 0;
1154                 }
1155
1156                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1157                         /* Handle deferred TIPs */
1158                         err = intel_pt_get_next_packet(decoder);
1159                         if (err)
1160                                 return err;
1161                         if (decoder->packet.type != INTEL_PT_TIP ||
1162                             decoder->packet.count == 0) {
1163                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1164                                                 decoder->ip);
1165                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1166                                 decoder->pkt_step = 0;
1167                                 return -ENOENT;
1168                         }
1169                         intel_pt_set_last_ip(decoder);
1170                         decoder->state.from_ip = decoder->ip;
1171                         decoder->state.to_ip = decoder->last_ip;
1172                         decoder->ip = decoder->last_ip;
1173                         return 0;
1174                 }
1175
1176                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1177                         decoder->tnt.count -= 1;
1178                         if (!decoder->tnt.count)
1179                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1180                         if (decoder->tnt.payload & BIT63) {
1181                                 decoder->tnt.payload <<= 1;
1182                                 decoder->state.from_ip = decoder->ip;
1183                                 decoder->ip += intel_pt_insn.length +
1184                                                intel_pt_insn.rel;
1185                                 decoder->state.to_ip = decoder->ip;
1186                                 return 0;
1187                         }
1188                         /* Instruction sample for a non-taken branch */
1189                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1190                                 decoder->tnt.payload <<= 1;
1191                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1192                                 decoder->state.from_ip = decoder->ip;
1193                                 decoder->state.to_ip = 0;
1194                                 decoder->ip += intel_pt_insn.length;
1195                                 return 0;
1196                         }
1197                         decoder->ip += intel_pt_insn.length;
1198                         if (!decoder->tnt.count)
1199                                 return -EAGAIN;
1200                         decoder->tnt.payload <<= 1;
1201                         continue;
1202                 }
1203
1204                 return intel_pt_bug(decoder);
1205         }
1206 }
1207
1208 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1209 {
1210         unsigned int fup_tx_flags;
1211         int err;
1212
1213         fup_tx_flags = decoder->packet.payload &
1214                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1215         err = intel_pt_get_next_packet(decoder);
1216         if (err)
1217                 return err;
1218         if (decoder->packet.type == INTEL_PT_FUP) {
1219                 decoder->fup_tx_flags = fup_tx_flags;
1220                 decoder->set_fup_tx_flags = true;
1221                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1222                         *no_tip = true;
1223         } else {
1224                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1225                                 decoder->pos);
1226                 intel_pt_update_in_tx(decoder);
1227         }
1228         return 0;
1229 }
1230
1231 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1232 {
1233         uint64_t timestamp;
1234
1235         decoder->have_tma = false;
1236
1237         if (decoder->ref_timestamp) {
1238                 timestamp = decoder->packet.payload |
1239                             (decoder->ref_timestamp & (0xffULL << 56));
1240                 if (timestamp < decoder->ref_timestamp) {
1241                         if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1242                                 timestamp += (1ULL << 56);
1243                 } else {
1244                         if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1245                                 timestamp -= (1ULL << 56);
1246                 }
1247                 decoder->tsc_timestamp = timestamp;
1248                 decoder->timestamp = timestamp;
1249                 decoder->ref_timestamp = 0;
1250                 decoder->timestamp_insn_cnt = 0;
1251         } else if (decoder->timestamp) {
1252                 timestamp = decoder->packet.payload |
1253                             (decoder->timestamp & (0xffULL << 56));
1254                 decoder->tsc_timestamp = timestamp;
1255                 if (timestamp < decoder->timestamp &&
1256                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1257                         intel_pt_log_to("Suppressing backwards timestamp",
1258                                         timestamp);
1259                         timestamp = decoder->timestamp;
1260                 }
1261                 if (timestamp < decoder->timestamp) {
1262                         intel_pt_log_to("Wraparound timestamp", timestamp);
1263                         timestamp += (1ULL << 56);
1264                         decoder->tsc_timestamp = timestamp;
1265                 }
1266                 decoder->timestamp = timestamp;
1267                 decoder->timestamp_insn_cnt = 0;
1268         }
1269
1270         if (decoder->last_packet_type == INTEL_PT_CYC) {
1271                 decoder->cyc_ref_timestamp = decoder->timestamp;
1272                 decoder->cycle_cnt = 0;
1273                 decoder->have_calc_cyc_to_tsc = false;
1274                 intel_pt_calc_cyc_to_tsc(decoder, false);
1275         }
1276
1277         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1278 }
1279
1280 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1281 {
1282         intel_pt_log("ERROR: Buffer overflow\n");
1283         intel_pt_clear_tx_flags(decoder);
1284         decoder->cbr = 0;
1285         decoder->timestamp_insn_cnt = 0;
1286         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1287         decoder->overflow = true;
1288         return -EOVERFLOW;
1289 }
1290
1291 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1292 {
1293         uint32_t ctc = decoder->packet.payload;
1294         uint32_t fc = decoder->packet.count;
1295         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1296
1297         if (!decoder->tsc_ctc_ratio_d)
1298                 return;
1299
1300         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1301         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1302         if (decoder->tsc_ctc_mult) {
1303                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1304         } else {
1305                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1306                                                   decoder->tsc_ctc_ratio_n,
1307                                                   decoder->tsc_ctc_ratio_d);
1308         }
1309         decoder->ctc_delta = 0;
1310         decoder->have_tma = true;
1311         decoder->fixup_last_mtc = true;
1312         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1313                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1314 }
1315
1316 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1317 {
1318         uint64_t timestamp;
1319         uint32_t mtc, mtc_delta;
1320
1321         if (!decoder->have_tma)
1322                 return;
1323
1324         mtc = decoder->packet.payload;
1325
1326         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1327                 decoder->fixup_last_mtc = false;
1328                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1329                                         &decoder->last_mtc);
1330         }
1331
1332         if (mtc > decoder->last_mtc)
1333                 mtc_delta = mtc - decoder->last_mtc;
1334         else
1335                 mtc_delta = mtc + 256 - decoder->last_mtc;
1336
1337         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1338
1339         if (decoder->tsc_ctc_mult) {
1340                 timestamp = decoder->ctc_timestamp +
1341                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1342         } else {
1343                 timestamp = decoder->ctc_timestamp +
1344                             multdiv(decoder->ctc_delta,
1345                                     decoder->tsc_ctc_ratio_n,
1346                                     decoder->tsc_ctc_ratio_d);
1347         }
1348
1349         if (timestamp < decoder->timestamp)
1350                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1351                              timestamp, decoder->timestamp);
1352         else
1353                 decoder->timestamp = timestamp;
1354
1355         decoder->timestamp_insn_cnt = 0;
1356         decoder->last_mtc = mtc;
1357
1358         if (decoder->last_packet_type == INTEL_PT_CYC) {
1359                 decoder->cyc_ref_timestamp = decoder->timestamp;
1360                 decoder->cycle_cnt = 0;
1361                 decoder->have_calc_cyc_to_tsc = false;
1362                 intel_pt_calc_cyc_to_tsc(decoder, true);
1363         }
1364 }
1365
1366 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1367 {
1368         unsigned int cbr = decoder->packet.payload;
1369
1370         if (decoder->cbr == cbr)
1371                 return;
1372
1373         decoder->cbr = cbr;
1374         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1375 }
1376
1377 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1378 {
1379         uint64_t timestamp = decoder->cyc_ref_timestamp;
1380
1381         decoder->have_cyc = true;
1382
1383         decoder->cycle_cnt += decoder->packet.payload;
1384
1385         if (!decoder->cyc_ref_timestamp)
1386                 return;
1387
1388         if (decoder->have_calc_cyc_to_tsc)
1389                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1390         else if (decoder->cbr)
1391                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1392         else
1393                 return;
1394
1395         if (timestamp < decoder->timestamp)
1396                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1397                              timestamp, decoder->timestamp);
1398         else
1399                 decoder->timestamp = timestamp;
1400
1401         decoder->timestamp_insn_cnt = 0;
1402 }
1403
1404 /* Walk PSB+ packets when already in sync. */
1405 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1406 {
1407         int err;
1408
1409         while (1) {
1410                 err = intel_pt_get_next_packet(decoder);
1411                 if (err)
1412                         return err;
1413
1414                 switch (decoder->packet.type) {
1415                 case INTEL_PT_PSBEND:
1416                         return 0;
1417
1418                 case INTEL_PT_TIP_PGD:
1419                 case INTEL_PT_TIP_PGE:
1420                 case INTEL_PT_TIP:
1421                 case INTEL_PT_TNT:
1422                 case INTEL_PT_TRACESTOP:
1423                 case INTEL_PT_BAD:
1424                 case INTEL_PT_PSB:
1425                         decoder->have_tma = false;
1426                         intel_pt_log("ERROR: Unexpected packet\n");
1427                         return -EAGAIN;
1428
1429                 case INTEL_PT_OVF:
1430                         return intel_pt_overflow(decoder);
1431
1432                 case INTEL_PT_TSC:
1433                         intel_pt_calc_tsc_timestamp(decoder);
1434                         break;
1435
1436                 case INTEL_PT_TMA:
1437                         intel_pt_calc_tma(decoder);
1438                         break;
1439
1440                 case INTEL_PT_CBR:
1441                         intel_pt_calc_cbr(decoder);
1442                         break;
1443
1444                 case INTEL_PT_MODE_EXEC:
1445                         decoder->exec_mode = decoder->packet.payload;
1446                         break;
1447
1448                 case INTEL_PT_PIP:
1449                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1450                         break;
1451
1452                 case INTEL_PT_FUP:
1453                         decoder->pge = true;
1454                         if (decoder->packet.count)
1455                                 intel_pt_set_last_ip(decoder);
1456                         break;
1457
1458                 case INTEL_PT_MODE_TSX:
1459                         intel_pt_update_in_tx(decoder);
1460                         break;
1461
1462                 case INTEL_PT_MTC:
1463                         intel_pt_calc_mtc_timestamp(decoder);
1464                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1465                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1466                         break;
1467
1468                 case INTEL_PT_CYC:
1469                 case INTEL_PT_VMCS:
1470                 case INTEL_PT_MNT:
1471                 case INTEL_PT_PAD:
1472                 default:
1473                         break;
1474                 }
1475         }
1476 }
1477
1478 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1479 {
1480         int err;
1481
1482         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1483                 decoder->tx_flags = 0;
1484                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1485                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1486         } else {
1487                 decoder->state.flags |= INTEL_PT_ASYNC;
1488         }
1489
1490         while (1) {
1491                 err = intel_pt_get_next_packet(decoder);
1492                 if (err)
1493                         return err;
1494
1495                 switch (decoder->packet.type) {
1496                 case INTEL_PT_TNT:
1497                 case INTEL_PT_FUP:
1498                 case INTEL_PT_TRACESTOP:
1499                 case INTEL_PT_PSB:
1500                 case INTEL_PT_TSC:
1501                 case INTEL_PT_TMA:
1502                 case INTEL_PT_MODE_TSX:
1503                 case INTEL_PT_BAD:
1504                 case INTEL_PT_PSBEND:
1505                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1506                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1507                         decoder->pkt_step = 0;
1508                         return -ENOENT;
1509
1510                 case INTEL_PT_CBR:
1511                         intel_pt_calc_cbr(decoder);
1512                         break;
1513
1514                 case INTEL_PT_OVF:
1515                         return intel_pt_overflow(decoder);
1516
1517                 case INTEL_PT_TIP_PGD:
1518                         decoder->state.from_ip = decoder->ip;
1519                         decoder->state.to_ip = 0;
1520                         if (decoder->packet.count != 0) {
1521                                 intel_pt_set_ip(decoder);
1522                                 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1523                                              decoder->ip);
1524                         }
1525                         decoder->pge = false;
1526                         decoder->continuous_period = false;
1527                         return 0;
1528
1529                 case INTEL_PT_TIP_PGE:
1530                         decoder->pge = true;
1531                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1532                                      decoder->ip);
1533                         decoder->state.from_ip = 0;
1534                         if (decoder->packet.count == 0) {
1535                                 decoder->state.to_ip = 0;
1536                         } else {
1537                                 intel_pt_set_ip(decoder);
1538                                 decoder->state.to_ip = decoder->ip;
1539                         }
1540                         return 0;
1541
1542                 case INTEL_PT_TIP:
1543                         decoder->state.from_ip = decoder->ip;
1544                         if (decoder->packet.count == 0) {
1545                                 decoder->state.to_ip = 0;
1546                         } else {
1547                                 intel_pt_set_ip(decoder);
1548                                 decoder->state.to_ip = decoder->ip;
1549                         }
1550                         return 0;
1551
1552                 case INTEL_PT_PIP:
1553                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1554                         break;
1555
1556                 case INTEL_PT_MTC:
1557                         intel_pt_calc_mtc_timestamp(decoder);
1558                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1559                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1560                         break;
1561
1562                 case INTEL_PT_CYC:
1563                         intel_pt_calc_cyc_timestamp(decoder);
1564                         break;
1565
1566                 case INTEL_PT_MODE_EXEC:
1567                         decoder->exec_mode = decoder->packet.payload;
1568                         break;
1569
1570                 case INTEL_PT_VMCS:
1571                 case INTEL_PT_MNT:
1572                 case INTEL_PT_PAD:
1573                         break;
1574
1575                 default:
1576                         return intel_pt_bug(decoder);
1577                 }
1578         }
1579 }
1580
1581 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1582 {
1583         bool no_tip = false;
1584         int err;
1585
1586         while (1) {
1587                 err = intel_pt_get_next_packet(decoder);
1588                 if (err)
1589                         return err;
1590 next:
1591                 switch (decoder->packet.type) {
1592                 case INTEL_PT_TNT:
1593                         if (!decoder->packet.count)
1594                                 break;
1595                         decoder->tnt = decoder->packet;
1596                         decoder->pkt_state = INTEL_PT_STATE_TNT;
1597                         err = intel_pt_walk_tnt(decoder);
1598                         if (err == -EAGAIN)
1599                                 break;
1600                         return err;
1601
1602                 case INTEL_PT_TIP_PGD:
1603                         if (decoder->packet.count != 0)
1604                                 intel_pt_set_last_ip(decoder);
1605                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1606                         return intel_pt_walk_tip(decoder);
1607
1608                 case INTEL_PT_TIP_PGE: {
1609                         decoder->pge = true;
1610                         if (decoder->packet.count == 0) {
1611                                 intel_pt_log_at("Skipping zero TIP.PGE",
1612                                                 decoder->pos);
1613                                 break;
1614                         }
1615                         intel_pt_set_ip(decoder);
1616                         decoder->state.from_ip = 0;
1617                         decoder->state.to_ip = decoder->ip;
1618                         return 0;
1619                 }
1620
1621                 case INTEL_PT_OVF:
1622                         return intel_pt_overflow(decoder);
1623
1624                 case INTEL_PT_TIP:
1625                         if (decoder->packet.count != 0)
1626                                 intel_pt_set_last_ip(decoder);
1627                         decoder->pkt_state = INTEL_PT_STATE_TIP;
1628                         return intel_pt_walk_tip(decoder);
1629
1630                 case INTEL_PT_FUP:
1631                         if (decoder->packet.count == 0) {
1632                                 intel_pt_log_at("Skipping zero FUP",
1633                                                 decoder->pos);
1634                                 no_tip = false;
1635                                 break;
1636                         }
1637                         intel_pt_set_last_ip(decoder);
1638                         err = intel_pt_walk_fup(decoder);
1639                         if (err != -EAGAIN) {
1640                                 if (err)
1641                                         return err;
1642                                 if (no_tip)
1643                                         decoder->pkt_state =
1644                                                 INTEL_PT_STATE_FUP_NO_TIP;
1645                                 else
1646                                         decoder->pkt_state = INTEL_PT_STATE_FUP;
1647                                 return 0;
1648                         }
1649                         if (no_tip) {
1650                                 no_tip = false;
1651                                 break;
1652                         }
1653                         return intel_pt_walk_fup_tip(decoder);
1654
1655                 case INTEL_PT_TRACESTOP:
1656                         decoder->pge = false;
1657                         decoder->continuous_period = false;
1658                         intel_pt_clear_tx_flags(decoder);
1659                         decoder->have_tma = false;
1660                         break;
1661
1662                 case INTEL_PT_PSB:
1663                         decoder->last_ip = 0;
1664                         decoder->have_last_ip = true;
1665                         intel_pt_clear_stack(&decoder->stack);
1666                         err = intel_pt_walk_psbend(decoder);
1667                         if (err == -EAGAIN)
1668                                 goto next;
1669                         if (err)
1670                                 return err;
1671                         break;
1672
1673                 case INTEL_PT_PIP:
1674                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1675                         break;
1676
1677                 case INTEL_PT_MTC:
1678                         intel_pt_calc_mtc_timestamp(decoder);
1679                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1680                                 break;
1681                         /*
1682                          * Ensure that there has been an instruction since the
1683                          * last MTC.
1684                          */
1685                         if (!decoder->mtc_insn)
1686                                 break;
1687                         decoder->mtc_insn = false;
1688                         /* Ensure that there is a timestamp */
1689                         if (!decoder->timestamp)
1690                                 break;
1691                         decoder->state.type = INTEL_PT_INSTRUCTION;
1692                         decoder->state.from_ip = decoder->ip;
1693                         decoder->state.to_ip = 0;
1694                         decoder->mtc_insn = false;
1695                         return 0;
1696
1697                 case INTEL_PT_TSC:
1698                         intel_pt_calc_tsc_timestamp(decoder);
1699                         break;
1700
1701                 case INTEL_PT_TMA:
1702                         intel_pt_calc_tma(decoder);
1703                         break;
1704
1705                 case INTEL_PT_CYC:
1706                         intel_pt_calc_cyc_timestamp(decoder);
1707                         break;
1708
1709                 case INTEL_PT_CBR:
1710                         intel_pt_calc_cbr(decoder);
1711                         break;
1712
1713                 case INTEL_PT_MODE_EXEC:
1714                         decoder->exec_mode = decoder->packet.payload;
1715                         break;
1716
1717                 case INTEL_PT_MODE_TSX:
1718                         /* MODE_TSX need not be followed by FUP */
1719                         if (!decoder->pge) {
1720                                 intel_pt_update_in_tx(decoder);
1721                                 break;
1722                         }
1723                         err = intel_pt_mode_tsx(decoder, &no_tip);
1724                         if (err)
1725                                 return err;
1726                         goto next;
1727
1728                 case INTEL_PT_BAD: /* Does not happen */
1729                         return intel_pt_bug(decoder);
1730
1731                 case INTEL_PT_PSBEND:
1732                 case INTEL_PT_VMCS:
1733                 case INTEL_PT_MNT:
1734                 case INTEL_PT_PAD:
1735                         break;
1736
1737                 default:
1738                         return intel_pt_bug(decoder);
1739                 }
1740         }
1741 }
1742
1743 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1744 {
1745         return decoder->packet.count &&
1746                (decoder->have_last_ip || decoder->packet.count == 3 ||
1747                 decoder->packet.count == 6);
1748 }
1749
1750 /* Walk PSB+ packets to get in sync. */
1751 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1752 {
1753         int err;
1754
1755         while (1) {
1756                 err = intel_pt_get_next_packet(decoder);
1757                 if (err)
1758                         return err;
1759
1760                 switch (decoder->packet.type) {
1761                 case INTEL_PT_TIP_PGD:
1762                         decoder->continuous_period = false;
1763                         __fallthrough;
1764                 case INTEL_PT_TIP_PGE:
1765                 case INTEL_PT_TIP:
1766                         intel_pt_log("ERROR: Unexpected packet\n");
1767                         return -ENOENT;
1768
1769                 case INTEL_PT_FUP:
1770                         decoder->pge = true;
1771                         if (intel_pt_have_ip(decoder)) {
1772                                 uint64_t current_ip = decoder->ip;
1773
1774                                 intel_pt_set_ip(decoder);
1775                                 if (current_ip)
1776                                         intel_pt_log_to("Setting IP",
1777                                                         decoder->ip);
1778                         }
1779                         break;
1780
1781                 case INTEL_PT_MTC:
1782                         intel_pt_calc_mtc_timestamp(decoder);
1783                         break;
1784
1785                 case INTEL_PT_TSC:
1786                         intel_pt_calc_tsc_timestamp(decoder);
1787                         break;
1788
1789                 case INTEL_PT_TMA:
1790                         intel_pt_calc_tma(decoder);
1791                         break;
1792
1793                 case INTEL_PT_CYC:
1794                         intel_pt_calc_cyc_timestamp(decoder);
1795                         break;
1796
1797                 case INTEL_PT_CBR:
1798                         intel_pt_calc_cbr(decoder);
1799                         break;
1800
1801                 case INTEL_PT_PIP:
1802                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1803                         break;
1804
1805                 case INTEL_PT_MODE_EXEC:
1806                         decoder->exec_mode = decoder->packet.payload;
1807                         break;
1808
1809                 case INTEL_PT_MODE_TSX:
1810                         intel_pt_update_in_tx(decoder);
1811                         break;
1812
1813                 case INTEL_PT_TRACESTOP:
1814                         decoder->pge = false;
1815                         decoder->continuous_period = false;
1816                         intel_pt_clear_tx_flags(decoder);
1817                         __fallthrough;
1818
1819                 case INTEL_PT_TNT:
1820                         decoder->have_tma = false;
1821                         intel_pt_log("ERROR: Unexpected packet\n");
1822                         if (decoder->ip)
1823                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
1824                         else
1825                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1826                         return -ENOENT;
1827
1828                 case INTEL_PT_BAD: /* Does not happen */
1829                         return intel_pt_bug(decoder);
1830
1831                 case INTEL_PT_OVF:
1832                         return intel_pt_overflow(decoder);
1833
1834                 case INTEL_PT_PSBEND:
1835                         return 0;
1836
1837                 case INTEL_PT_PSB:
1838                 case INTEL_PT_VMCS:
1839                 case INTEL_PT_MNT:
1840                 case INTEL_PT_PAD:
1841                 default:
1842                         break;
1843                 }
1844         }
1845 }
1846
1847 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1848 {
1849         int err;
1850
1851         while (1) {
1852                 err = intel_pt_get_next_packet(decoder);
1853                 if (err)
1854                         return err;
1855
1856                 switch (decoder->packet.type) {
1857                 case INTEL_PT_TIP_PGD:
1858                         decoder->continuous_period = false;
1859                         __fallthrough;
1860                 case INTEL_PT_TIP_PGE:
1861                 case INTEL_PT_TIP:
1862                         decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1863                         if (intel_pt_have_ip(decoder))
1864                                 intel_pt_set_ip(decoder);
1865                         if (decoder->ip)
1866                                 return 0;
1867                         break;
1868
1869                 case INTEL_PT_FUP:
1870                         if (intel_pt_have_ip(decoder))
1871                                 intel_pt_set_ip(decoder);
1872                         if (decoder->ip)
1873                                 return 0;
1874                         break;
1875
1876                 case INTEL_PT_MTC:
1877                         intel_pt_calc_mtc_timestamp(decoder);
1878                         break;
1879
1880                 case INTEL_PT_TSC:
1881                         intel_pt_calc_tsc_timestamp(decoder);
1882                         break;
1883
1884                 case INTEL_PT_TMA:
1885                         intel_pt_calc_tma(decoder);
1886                         break;
1887
1888                 case INTEL_PT_CYC:
1889                         intel_pt_calc_cyc_timestamp(decoder);
1890                         break;
1891
1892                 case INTEL_PT_CBR:
1893                         intel_pt_calc_cbr(decoder);
1894                         break;
1895
1896                 case INTEL_PT_PIP:
1897                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1898                         break;
1899
1900                 case INTEL_PT_MODE_EXEC:
1901                         decoder->exec_mode = decoder->packet.payload;
1902                         break;
1903
1904                 case INTEL_PT_MODE_TSX:
1905                         intel_pt_update_in_tx(decoder);
1906                         break;
1907
1908                 case INTEL_PT_OVF:
1909                         return intel_pt_overflow(decoder);
1910
1911                 case INTEL_PT_BAD: /* Does not happen */
1912                         return intel_pt_bug(decoder);
1913
1914                 case INTEL_PT_TRACESTOP:
1915                         decoder->pge = false;
1916                         decoder->continuous_period = false;
1917                         intel_pt_clear_tx_flags(decoder);
1918                         decoder->have_tma = false;
1919                         break;
1920
1921                 case INTEL_PT_PSB:
1922                         decoder->last_ip = 0;
1923                         decoder->have_last_ip = true;
1924                         intel_pt_clear_stack(&decoder->stack);
1925                         err = intel_pt_walk_psb(decoder);
1926                         if (err)
1927                                 return err;
1928                         if (decoder->ip) {
1929                                 /* Do not have a sample */
1930                                 decoder->state.type = 0;
1931                                 return 0;
1932                         }
1933                         break;
1934
1935                 case INTEL_PT_TNT:
1936                 case INTEL_PT_PSBEND:
1937                 case INTEL_PT_VMCS:
1938                 case INTEL_PT_MNT:
1939                 case INTEL_PT_PAD:
1940                 default:
1941                         break;
1942                 }
1943         }
1944 }
1945
1946 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1947 {
1948         int err;
1949
1950         decoder->set_fup_tx_flags = false;
1951
1952         intel_pt_log("Scanning for full IP\n");
1953         err = intel_pt_walk_to_ip(decoder);
1954         if (err)
1955                 return err;
1956
1957         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1958         decoder->overflow = false;
1959
1960         decoder->state.from_ip = 0;
1961         decoder->state.to_ip = decoder->ip;
1962         intel_pt_log_to("Setting IP", decoder->ip);
1963
1964         return 0;
1965 }
1966
1967 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
1968 {
1969         const unsigned char *end = decoder->buf + decoder->len;
1970         size_t i;
1971
1972         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
1973                 if (i > decoder->len)
1974                         continue;
1975                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
1976                         return i;
1977         }
1978         return 0;
1979 }
1980
1981 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
1982 {
1983         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
1984         const char *psb = INTEL_PT_PSB_STR;
1985
1986         if (rest_psb > decoder->len ||
1987             memcmp(decoder->buf, psb + part_psb, rest_psb))
1988                 return 0;
1989
1990         return rest_psb;
1991 }
1992
1993 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
1994                                   int part_psb)
1995 {
1996         int rest_psb, ret;
1997
1998         decoder->pos += decoder->len;
1999         decoder->len = 0;
2000
2001         ret = intel_pt_get_next_data(decoder);
2002         if (ret)
2003                 return ret;
2004
2005         rest_psb = intel_pt_rest_psb(decoder, part_psb);
2006         if (!rest_psb)
2007                 return 0;
2008
2009         decoder->pos -= part_psb;
2010         decoder->next_buf = decoder->buf + rest_psb;
2011         decoder->next_len = decoder->len - rest_psb;
2012         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2013         decoder->buf = decoder->temp_buf;
2014         decoder->len = INTEL_PT_PSB_LEN;
2015
2016         return 0;
2017 }
2018
2019 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2020 {
2021         unsigned char *next;
2022         int ret;
2023
2024         intel_pt_log("Scanning for PSB\n");
2025         while (1) {
2026                 if (!decoder->len) {
2027                         ret = intel_pt_get_next_data(decoder);
2028                         if (ret)
2029                                 return ret;
2030                 }
2031
2032                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2033                               INTEL_PT_PSB_LEN);
2034                 if (!next) {
2035                         int part_psb;
2036
2037                         part_psb = intel_pt_part_psb(decoder);
2038                         if (part_psb) {
2039                                 ret = intel_pt_get_split_psb(decoder, part_psb);
2040                                 if (ret)
2041                                         return ret;
2042                         } else {
2043                                 decoder->pos += decoder->len;
2044                                 decoder->len = 0;
2045                         }
2046                         continue;
2047                 }
2048
2049                 decoder->pkt_step = next - decoder->buf;
2050                 return intel_pt_get_next_packet(decoder);
2051         }
2052 }
2053
2054 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2055 {
2056         int err;
2057
2058         decoder->pge = false;
2059         decoder->continuous_period = false;
2060         decoder->have_last_ip = false;
2061         decoder->last_ip = 0;
2062         decoder->ip = 0;
2063         intel_pt_clear_stack(&decoder->stack);
2064
2065         err = intel_pt_scan_for_psb(decoder);
2066         if (err)
2067                 return err;
2068
2069         decoder->have_last_ip = true;
2070         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2071
2072         err = intel_pt_walk_psb(decoder);
2073         if (err)
2074                 return err;
2075
2076         if (decoder->ip) {
2077                 decoder->state.type = 0; /* Do not have a sample */
2078                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2079         } else {
2080                 return intel_pt_sync_ip(decoder);
2081         }
2082
2083         return 0;
2084 }
2085
2086 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2087 {
2088         uint64_t est = decoder->sample_insn_cnt << 1;
2089
2090         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2091                 goto out;
2092
2093         est *= decoder->max_non_turbo_ratio;
2094         est /= decoder->cbr;
2095 out:
2096         return decoder->sample_timestamp + est;
2097 }
2098
2099 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2100 {
2101         int err;
2102
2103         do {
2104                 decoder->state.type = INTEL_PT_BRANCH;
2105                 decoder->state.flags = 0;
2106
2107                 switch (decoder->pkt_state) {
2108                 case INTEL_PT_STATE_NO_PSB:
2109                         err = intel_pt_sync(decoder);
2110                         break;
2111                 case INTEL_PT_STATE_NO_IP:
2112                         decoder->have_last_ip = false;
2113                         decoder->last_ip = 0;
2114                         decoder->ip = 0;
2115                         /* Fall through */
2116                 case INTEL_PT_STATE_ERR_RESYNC:
2117                         err = intel_pt_sync_ip(decoder);
2118                         break;
2119                 case INTEL_PT_STATE_IN_SYNC:
2120                         err = intel_pt_walk_trace(decoder);
2121                         break;
2122                 case INTEL_PT_STATE_TNT:
2123                         err = intel_pt_walk_tnt(decoder);
2124                         if (err == -EAGAIN)
2125                                 err = intel_pt_walk_trace(decoder);
2126                         break;
2127                 case INTEL_PT_STATE_TIP:
2128                 case INTEL_PT_STATE_TIP_PGD:
2129                         err = intel_pt_walk_tip(decoder);
2130                         break;
2131                 case INTEL_PT_STATE_FUP:
2132                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2133                         err = intel_pt_walk_fup(decoder);
2134                         if (err == -EAGAIN)
2135                                 err = intel_pt_walk_fup_tip(decoder);
2136                         else if (!err)
2137                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
2138                         break;
2139                 case INTEL_PT_STATE_FUP_NO_TIP:
2140                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2141                         err = intel_pt_walk_fup(decoder);
2142                         if (err == -EAGAIN)
2143                                 err = intel_pt_walk_trace(decoder);
2144                         break;
2145                 default:
2146                         err = intel_pt_bug(decoder);
2147                         break;
2148                 }
2149         } while (err == -ENOLINK);
2150
2151         if (err) {
2152                 decoder->state.err = intel_pt_ext_err(err);
2153                 decoder->state.from_ip = decoder->ip;
2154                 decoder->sample_timestamp = decoder->timestamp;
2155                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2156         } else {
2157                 decoder->state.err = 0;
2158                 if (intel_pt_sample_time(decoder->pkt_state)) {
2159                         decoder->sample_timestamp = decoder->timestamp;
2160                         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2161                 }
2162         }
2163
2164         decoder->state.timestamp = decoder->sample_timestamp;
2165         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2166         decoder->state.cr3 = decoder->cr3;
2167         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2168
2169         return &decoder->state;
2170 }
2171
2172 /**
2173  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2174  * @buf: pointer to buffer pointer
2175  * @len: size of buffer
2176  *
2177  * Updates the buffer pointer to point to the start of the next PSB packet if
2178  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2179  * @len is adjusted accordingly.
2180  *
2181  * Return: %true if a PSB packet is found, %false otherwise.
2182  */
2183 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2184 {
2185         unsigned char *next;
2186
2187         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2188         if (next) {
2189                 *len -= next - *buf;
2190                 *buf = next;
2191                 return true;
2192         }
2193         return false;
2194 }
2195
2196 /**
2197  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2198  *                     packet.
2199  * @buf: pointer to buffer pointer
2200  * @len: size of buffer
2201  *
2202  * Updates the buffer pointer to point to the start of the following PSB packet
2203  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2204  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2205  *
2206  * Return: %true if a PSB packet is found, %false otherwise.
2207  */
2208 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2209 {
2210         unsigned char *next;
2211
2212         if (!*len)
2213                 return false;
2214
2215         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2216         if (next) {
2217                 *len -= next - *buf;
2218                 *buf = next;
2219                 return true;
2220         }
2221         return false;
2222 }
2223
2224 /**
2225  * intel_pt_last_psb - find the last PSB packet in a buffer.
2226  * @buf: buffer
2227  * @len: size of buffer
2228  *
2229  * This function finds the last PSB in a buffer.
2230  *
2231  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2232  */
2233 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2234 {
2235         const char *n = INTEL_PT_PSB_STR;
2236         unsigned char *p;
2237         size_t k;
2238
2239         if (len < INTEL_PT_PSB_LEN)
2240                 return NULL;
2241
2242         k = len - INTEL_PT_PSB_LEN + 1;
2243         while (1) {
2244                 p = memrchr(buf, n[0], k);
2245                 if (!p)
2246                         return NULL;
2247                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2248                         return p;
2249                 k = p - buf;
2250                 if (!k)
2251                         return NULL;
2252         }
2253 }
2254
2255 /**
2256  * intel_pt_next_tsc - find and return next TSC.
2257  * @buf: buffer
2258  * @len: size of buffer
2259  * @tsc: TSC value returned
2260  * @rem: returns remaining size when TSC is found
2261  *
2262  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2263  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2264  * PSBEND packet is found.
2265  *
2266  * Return: %true if TSC is found, false otherwise.
2267  */
2268 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
2269                               size_t *rem)
2270 {
2271         struct intel_pt_pkt packet;
2272         int ret;
2273
2274         while (len) {
2275                 ret = intel_pt_get_packet(buf, len, &packet);
2276                 if (ret <= 0)
2277                         return false;
2278                 if (packet.type == INTEL_PT_TSC) {
2279                         *tsc = packet.payload;
2280                         *rem = len;
2281                         return true;
2282                 }
2283                 if (packet.type == INTEL_PT_PSBEND)
2284                         return false;
2285                 buf += ret;
2286                 len -= ret;
2287         }
2288         return false;
2289 }
2290
2291 /**
2292  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2293  * @tsc1: first TSC to compare
2294  * @tsc2: second TSC to compare
2295  *
2296  * This function compares 7-byte TSC values allowing for the possibility that
2297  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2298  * around so for that purpose this function assumes the absolute difference is
2299  * less than half the maximum difference.
2300  *
2301  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2302  * after @tsc2.
2303  */
2304 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2305 {
2306         const uint64_t halfway = (1ULL << 55);
2307
2308         if (tsc1 == tsc2)
2309                 return 0;
2310
2311         if (tsc1 < tsc2) {
2312                 if (tsc2 - tsc1 < halfway)
2313                         return -1;
2314                 else
2315                         return 1;
2316         } else {
2317                 if (tsc1 - tsc2 < halfway)
2318                         return 1;
2319                 else
2320                         return -1;
2321         }
2322 }
2323
2324 /**
2325  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2326  *                             using TSC.
2327  * @buf_a: first buffer
2328  * @len_a: size of first buffer
2329  * @buf_b: second buffer
2330  * @len_b: size of second buffer
2331  * @consecutive: returns true if there is data in buf_b that is consecutive
2332  *               to buf_a
2333  *
2334  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2335  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2336  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2337  * @buf_a and @buf_b are positioned at a PSB.
2338  *
2339  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2340  * @buf_b + @len_b if there is no non-overlapped data.
2341  */
2342 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2343                                                 size_t len_a,
2344                                                 unsigned char *buf_b,
2345                                                 size_t len_b, bool *consecutive)
2346 {
2347         uint64_t tsc_a, tsc_b;
2348         unsigned char *p;
2349         size_t len, rem_a, rem_b;
2350
2351         p = intel_pt_last_psb(buf_a, len_a);
2352         if (!p)
2353                 return buf_b; /* No PSB in buf_a => no overlap */
2354
2355         len = len_a - (p - buf_a);
2356         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
2357                 /* The last PSB+ in buf_a is incomplete, so go back one more */
2358                 len_a -= len;
2359                 p = intel_pt_last_psb(buf_a, len_a);
2360                 if (!p)
2361                         return buf_b; /* No full PSB+ => assume no overlap */
2362                 len = len_a - (p - buf_a);
2363                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
2364                         return buf_b; /* No TSC in buf_a => assume no overlap */
2365         }
2366
2367         while (1) {
2368                 /* Ignore PSB+ with no TSC */
2369                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
2370                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
2371
2372                         /* Same TSC, so buffers are consecutive */
2373                         if (!cmp && rem_b >= rem_a) {
2374                                 *consecutive = true;
2375                                 return buf_b + len_b - (rem_b - rem_a);
2376                         }
2377                         if (cmp < 0)
2378                                 return buf_b; /* tsc_a < tsc_b => no overlap */
2379                 }
2380
2381                 if (!intel_pt_step_psb(&buf_b, &len_b))
2382                         return buf_b + len_b; /* No PSB in buf_b => no data */
2383         }
2384 }
2385
2386 /**
2387  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2388  * @buf_a: first buffer
2389  * @len_a: size of first buffer
2390  * @buf_b: second buffer
2391  * @len_b: size of second buffer
2392  * @have_tsc: can use TSC packets to detect overlap
2393  * @consecutive: returns true if there is data in buf_b that is consecutive
2394  *               to buf_a
2395  *
2396  * When trace samples or snapshots are recorded there is the possibility that
2397  * the data overlaps.  Note that, for the purposes of decoding, data is only
2398  * useful if it begins with a PSB packet.
2399  *
2400  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2401  * @buf_b + @len_b if there is no non-overlapped data.
2402  */
2403 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2404                                      unsigned char *buf_b, size_t len_b,
2405                                      bool have_tsc, bool *consecutive)
2406 {
2407         unsigned char *found;
2408
2409         /* Buffer 'b' must start at PSB so throw away everything before that */
2410         if (!intel_pt_next_psb(&buf_b, &len_b))
2411                 return buf_b + len_b; /* No PSB */
2412
2413         if (!intel_pt_next_psb(&buf_a, &len_a))
2414                 return buf_b; /* No overlap */
2415
2416         if (have_tsc) {
2417                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
2418                                                   consecutive);
2419                 if (found)
2420                         return found;
2421         }
2422
2423         /*
2424          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2425          * we can ignore the first part of buffer 'a'.
2426          */
2427         while (len_b < len_a) {
2428                 if (!intel_pt_step_psb(&buf_a, &len_a))
2429                         return buf_b; /* No overlap */
2430         }
2431
2432         /* Now len_b >= len_a */
2433         while (1) {
2434                 /* Potential overlap so check the bytes */
2435                 found = memmem(buf_a, len_a, buf_b, len_a);
2436                 if (found) {
2437                         *consecutive = true;
2438                         return buf_b + len_a;
2439                 }
2440
2441                 /* Try again at next PSB in buffer 'a' */
2442                 if (!intel_pt_step_psb(&buf_a, &len_a))
2443                         return buf_b; /* No overlap */
2444         }
2445 }