1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
7 #include <linux/atomic.h>
8 #include <linux/circ_buf.h>
9 #include <linux/coresight.h>
10 #include <linux/perf_event.h>
11 #include <linux/slab.h>
12 #include "coresight-priv.h"
13 #include "coresight-tmc.h"
14 #include "coresight-etm-perf.h"
16 static int tmc_set_etf_buffer(struct coresight_device *csdev,
17 struct perf_output_handle *handle);
19 static int __tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
23 CS_UNLOCK(drvdata->base);
25 /* Wait for TMCSReady bit to be set */
26 rc = tmc_wait_for_tmcready(drvdata);
28 dev_err(&drvdata->csdev->dev,
29 "Failed to enable: TMC not ready\n");
30 CS_LOCK(drvdata->base);
34 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
35 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
36 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
37 TMC_FFCR_TRIGON_TRIGIN,
38 drvdata->base + TMC_FFCR);
40 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
41 tmc_enable_hw(drvdata);
43 CS_LOCK(drvdata->base);
47 static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
49 int rc = coresight_claim_device(drvdata->csdev);
54 rc = __tmc_etb_enable_hw(drvdata);
56 coresight_disclaim_device(drvdata->csdev);
60 static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
65 /* Check if the buffer wrapped around. */
66 lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL;
70 read_data = readl_relaxed(drvdata->base + TMC_RRD);
71 if (read_data == 0xFFFFFFFF)
73 memcpy(bufp, &read_data, 4);
79 coresight_insert_barrier_packet(drvdata->buf);
83 static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
85 CS_UNLOCK(drvdata->base);
87 tmc_flush_and_stop(drvdata);
89 * When operating in sysFS mode the content of the buffer needs to be
90 * read before the TMC is disabled.
92 if (drvdata->mode == CS_MODE_SYSFS)
93 tmc_etb_dump_hw(drvdata);
94 tmc_disable_hw(drvdata);
96 CS_LOCK(drvdata->base);
99 static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
101 __tmc_etb_disable_hw(drvdata);
102 coresight_disclaim_device(drvdata->csdev);
105 static int __tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
109 CS_UNLOCK(drvdata->base);
111 /* Wait for TMCSReady bit to be set */
112 rc = tmc_wait_for_tmcready(drvdata);
114 dev_err(&drvdata->csdev->dev,
115 "Failed to enable : TMC is not ready\n");
116 CS_LOCK(drvdata->base);
120 writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);
121 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,
122 drvdata->base + TMC_FFCR);
123 writel_relaxed(0x0, drvdata->base + TMC_BUFWM);
124 tmc_enable_hw(drvdata);
126 CS_LOCK(drvdata->base);
130 static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
132 int rc = coresight_claim_device(drvdata->csdev);
137 rc = __tmc_etf_enable_hw(drvdata);
139 coresight_disclaim_device(drvdata->csdev);
143 static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
145 struct coresight_device *csdev = drvdata->csdev;
147 CS_UNLOCK(drvdata->base);
149 tmc_flush_and_stop(drvdata);
150 tmc_disable_hw(drvdata);
151 coresight_disclaim_device_unlocked(csdev);
152 CS_LOCK(drvdata->base);
156 * Return the available trace data in the buffer from @pos, with
157 * a maximum limit of @len, updating the @bufpp on where to
160 ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata,
161 loff_t pos, size_t len, char **bufpp)
163 ssize_t actual = len;
165 /* Adjust the len to available size @pos */
166 if (pos + actual > drvdata->len)
167 actual = drvdata->len - pos;
169 *bufpp = drvdata->buf + pos;
173 static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)
179 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
182 * If we don't have a buffer release the lock and allocate memory.
183 * Otherwise keep the lock and move along.
185 spin_lock_irqsave(&drvdata->spinlock, flags);
187 spin_unlock_irqrestore(&drvdata->spinlock, flags);
189 /* Allocating the memory here while outside of the spinlock */
190 buf = kzalloc(drvdata->size, GFP_KERNEL);
194 /* Let's try again */
195 spin_lock_irqsave(&drvdata->spinlock, flags);
198 if (drvdata->reading) {
204 * In sysFS mode we can have multiple writers per sink. Since this
205 * sink is already enabled no memory is needed and the HW need not be
208 if (drvdata->mode == CS_MODE_SYSFS) {
209 atomic_inc(&csdev->refcnt);
214 * If drvdata::buf isn't NULL, memory was allocated for a previous
215 * trace run but wasn't read. If so simply zero-out the memory.
216 * Otherwise use the memory allocated above.
218 * The memory is freed when users read the buffer using the
219 * /dev/xyz.{etf|etb} interface. See tmc_read_unprepare_etf() for
223 memset(drvdata->buf, 0, drvdata->size);
229 ret = tmc_etb_enable_hw(drvdata);
231 drvdata->mode = CS_MODE_SYSFS;
232 atomic_inc(&csdev->refcnt);
234 /* Free up the buffer if we failed to enable */
238 spin_unlock_irqrestore(&drvdata->spinlock, flags);
240 /* Free memory outside the spinlock if need be */
247 static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data)
252 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
253 struct perf_output_handle *handle = data;
254 struct cs_buffers *buf = etm_perf_sink_config(handle);
256 spin_lock_irqsave(&drvdata->spinlock, flags);
259 if (drvdata->reading)
262 * No need to continue if the ETB/ETF is already operated
265 if (drvdata->mode == CS_MODE_SYSFS) {
270 /* Get a handle on the pid of the process to monitor */
273 if (drvdata->pid != -1 && drvdata->pid != pid) {
278 ret = tmc_set_etf_buffer(csdev, handle);
283 * No HW configuration is needed if the sink is already in
284 * use for this session.
286 if (drvdata->pid == pid) {
287 atomic_inc(&csdev->refcnt);
291 ret = tmc_etb_enable_hw(drvdata);
293 /* Associate with monitored process. */
295 drvdata->mode = CS_MODE_PERF;
296 atomic_inc(&csdev->refcnt);
299 spin_unlock_irqrestore(&drvdata->spinlock, flags);
304 static int tmc_enable_etf_sink(struct coresight_device *csdev,
305 enum cs_mode mode, void *data)
311 ret = tmc_enable_etf_sink_sysfs(csdev);
314 ret = tmc_enable_etf_sink_perf(csdev, data);
316 /* We shouldn't be here */
325 dev_dbg(&csdev->dev, "TMC-ETB/ETF enabled\n");
329 static int tmc_disable_etf_sink(struct coresight_device *csdev)
332 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
334 spin_lock_irqsave(&drvdata->spinlock, flags);
336 if (drvdata->reading) {
337 spin_unlock_irqrestore(&drvdata->spinlock, flags);
341 if (atomic_dec_return(&csdev->refcnt)) {
342 spin_unlock_irqrestore(&drvdata->spinlock, flags);
346 /* Complain if we (somehow) got out of sync */
347 WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
348 tmc_etb_disable_hw(drvdata);
349 /* Dissociate from monitored process. */
351 drvdata->mode = CS_MODE_DISABLED;
353 spin_unlock_irqrestore(&drvdata->spinlock, flags);
355 dev_dbg(&csdev->dev, "TMC-ETB/ETF disabled\n");
359 static int tmc_enable_etf_link(struct coresight_device *csdev,
360 struct coresight_connection *in,
361 struct coresight_connection *out)
365 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
366 bool first_enable = false;
368 spin_lock_irqsave(&drvdata->spinlock, flags);
369 if (drvdata->reading) {
370 spin_unlock_irqrestore(&drvdata->spinlock, flags);
374 if (atomic_read(&csdev->refcnt) == 0) {
375 ret = tmc_etf_enable_hw(drvdata);
377 drvdata->mode = CS_MODE_SYSFS;
382 atomic_inc(&csdev->refcnt);
383 spin_unlock_irqrestore(&drvdata->spinlock, flags);
386 dev_dbg(&csdev->dev, "TMC-ETF enabled\n");
390 static void tmc_disable_etf_link(struct coresight_device *csdev,
391 struct coresight_connection *in,
392 struct coresight_connection *out)
395 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
396 bool last_disable = false;
398 spin_lock_irqsave(&drvdata->spinlock, flags);
399 if (drvdata->reading) {
400 spin_unlock_irqrestore(&drvdata->spinlock, flags);
404 if (atomic_dec_return(&csdev->refcnt) == 0) {
405 tmc_etf_disable_hw(drvdata);
406 drvdata->mode = CS_MODE_DISABLED;
409 spin_unlock_irqrestore(&drvdata->spinlock, flags);
412 dev_dbg(&csdev->dev, "TMC-ETF disabled\n");
415 static void *tmc_alloc_etf_buffer(struct coresight_device *csdev,
416 struct perf_event *event, void **pages,
417 int nr_pages, bool overwrite)
420 struct cs_buffers *buf;
422 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
424 /* Allocate memory structure for interaction with Perf */
425 buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);
429 buf->pid = task_pid_nr(event->owner);
430 buf->snapshot = overwrite;
431 buf->nr_pages = nr_pages;
432 buf->data_pages = pages;
437 static void tmc_free_etf_buffer(void *config)
439 struct cs_buffers *buf = config;
444 static int tmc_set_etf_buffer(struct coresight_device *csdev,
445 struct perf_output_handle *handle)
449 struct cs_buffers *buf = etm_perf_sink_config(handle);
454 /* wrap head around to the amount of space we have */
455 head = handle->head & (((unsigned long)buf->nr_pages << PAGE_SHIFT) - 1);
457 /* find the page to write to */
458 buf->cur = head / PAGE_SIZE;
460 /* and offset within that page */
461 buf->offset = head % PAGE_SIZE;
463 local_set(&buf->data_size, 0);
468 static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev,
469 struct perf_output_handle *handle,
476 u64 read_ptr, write_ptr;
478 unsigned long offset, to_read = 0, flags;
479 struct cs_buffers *buf = sink_config;
480 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
485 /* This shouldn't happen */
486 if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF))
489 spin_lock_irqsave(&drvdata->spinlock, flags);
491 /* Don't do anything if another tracer is using this sink */
492 if (atomic_read(&csdev->refcnt) != 1)
495 CS_UNLOCK(drvdata->base);
497 tmc_flush_and_stop(drvdata);
499 read_ptr = tmc_read_rrp(drvdata);
500 write_ptr = tmc_read_rwp(drvdata);
503 * Get a hold of the status register and see if a wrap around
504 * has occurred. If so adjust things accordingly.
506 status = readl_relaxed(drvdata->base + TMC_STS);
507 if (status & TMC_STS_FULL) {
509 to_read = drvdata->size;
511 to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
515 * The TMC RAM buffer may be bigger than the space available in the
516 * perf ring buffer (handle->size). If so advance the RRP so that we
517 * get the latest trace data. In snapshot mode none of that matters
518 * since we are expected to clobber stale data in favour of the latest
521 if (!buf->snapshot && to_read > handle->size) {
522 u32 mask = tmc_get_memwidth_mask(drvdata);
525 * Make sure the new size is aligned in accordance with the
526 * requirement explained in function tmc_get_memwidth_mask().
528 to_read = handle->size & mask;
529 /* Move the RAM read pointer up */
530 read_ptr = (write_ptr + drvdata->size) - to_read;
531 /* Make sure we are still within our limits */
532 if (read_ptr > (drvdata->size - 1))
533 read_ptr -= drvdata->size;
535 tmc_write_rrp(drvdata, read_ptr);
540 * Don't set the TRUNCATED flag in snapshot mode because 1) the
541 * captured buffer is expected to be truncated and 2) a full buffer
542 * prevents the event from being re-enabled by the perf core,
543 * resulting in stale data being send to user space.
545 if (!buf->snapshot && lost)
546 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
549 offset = buf->offset;
550 barrier = coresight_barrier_pkt;
552 /* for every byte to read */
553 for (i = 0; i < to_read; i += 4) {
554 buf_ptr = buf->data_pages[cur] + offset;
555 *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
557 if (lost && i < CORESIGHT_BARRIER_PKT_SIZE) {
563 if (offset >= PAGE_SIZE) {
566 /* wrap around at the end of the buffer */
567 cur &= buf->nr_pages - 1;
572 * In snapshot mode we simply increment the head by the number of byte
573 * that were written. User space will figure out how many bytes to get
574 * from the AUX buffer based on the position of the head.
577 handle->head += to_read;
580 * CS_LOCK() contains mb() so it can ensure visibility of the AUX trace
581 * data before the aux_head is updated via perf_aux_output_end(), which
582 * is expected by the perf ring buffer.
584 CS_LOCK(drvdata->base);
586 spin_unlock_irqrestore(&drvdata->spinlock, flags);
591 static const struct coresight_ops_sink tmc_etf_sink_ops = {
592 .enable = tmc_enable_etf_sink,
593 .disable = tmc_disable_etf_sink,
594 .alloc_buffer = tmc_alloc_etf_buffer,
595 .free_buffer = tmc_free_etf_buffer,
596 .update_buffer = tmc_update_etf_buffer,
599 static const struct coresight_ops_link tmc_etf_link_ops = {
600 .enable = tmc_enable_etf_link,
601 .disable = tmc_disable_etf_link,
604 const struct coresight_ops tmc_etb_cs_ops = {
605 .sink_ops = &tmc_etf_sink_ops,
608 const struct coresight_ops tmc_etf_cs_ops = {
609 .sink_ops = &tmc_etf_sink_ops,
610 .link_ops = &tmc_etf_link_ops,
613 int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
619 /* config types are set a boot time and never change */
620 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
621 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
624 spin_lock_irqsave(&drvdata->spinlock, flags);
626 if (drvdata->reading) {
631 /* Don't interfere if operated from Perf */
632 if (drvdata->mode == CS_MODE_PERF) {
637 /* If drvdata::buf is NULL the trace data has been read already */
638 if (drvdata->buf == NULL) {
643 /* Disable the TMC if need be */
644 if (drvdata->mode == CS_MODE_SYSFS) {
645 /* There is no point in reading a TMC in HW FIFO mode */
646 mode = readl_relaxed(drvdata->base + TMC_MODE);
647 if (mode != TMC_MODE_CIRCULAR_BUFFER) {
651 __tmc_etb_disable_hw(drvdata);
654 drvdata->reading = true;
656 spin_unlock_irqrestore(&drvdata->spinlock, flags);
661 int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
668 /* config types are set a boot time and never change */
669 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
670 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
673 spin_lock_irqsave(&drvdata->spinlock, flags);
675 /* Re-enable the TMC if need be */
676 if (drvdata->mode == CS_MODE_SYSFS) {
677 /* There is no point in reading a TMC in HW FIFO mode */
678 mode = readl_relaxed(drvdata->base + TMC_MODE);
679 if (mode != TMC_MODE_CIRCULAR_BUFFER) {
680 spin_unlock_irqrestore(&drvdata->spinlock, flags);
684 * The trace run will continue with the same allocated trace
685 * buffer. As such zero-out the buffer so that we don't end
686 * up with stale data.
688 * Since the tracer is still enabled drvdata::buf
691 memset(drvdata->buf, 0, drvdata->size);
692 rc = __tmc_etb_enable_hw(drvdata);
694 spin_unlock_irqrestore(&drvdata->spinlock, flags);
699 * The ETB/ETF is not tracing and the buffer was just read.
700 * As such prepare to free the trace buffer.
706 drvdata->reading = false;
707 spin_unlock_irqrestore(&drvdata->spinlock, flags);
710 * Free allocated memory outside of the spinlock. There is no need
711 * to assert the validity of 'buf' since calling kfree(NULL) is safe.