Star8
71
代码
代码
Issues36
Pull Requests13
项目讨论
Wiki
分析
Star8
71
已合并
6.1CVE漏洞 #305
何京晶创建于 3月5日
6.1CVE漏洞 #305
已合并
何京晶创建于 3月5日
从Hun_Dun/kernel_linux_6.6hjj_cve_6.1_2合入到openharmony/kernel_linux_6.6OpenHarmony-6.1-Release
共 32 个文件变更+4952-4257
March/mips/sgi-ip22/ip22-platform.c+32-0
@@ -221,3 +221,35 @@ static int __init sgi_ds1286_devinit(void)
221}221}
222 222 
223device_initcall(sgi_ds1286_devinit);223device_initcall(sgi_ds1286_devinit);
224+ 
225+#define SGI_ZILOG_BASE (HPC3_CHIP0_BASE + \
226+ offsetof(struct hpc3_regs, pbus_extregs[6]) + \
227+ offsetof(struct sgioc_regs, uart))
228+ 
229+static struct resource sgi_zilog_resources[] = {
230+ {
231+ .start = SGI_ZILOG_BASE,
232+ .end = SGI_ZILOG_BASE + 15,
233+ .flags = IORESOURCE_MEM
234+ },
235+ {
236+ .start = SGI_SERIAL_IRQ,
237+ .end = SGI_SERIAL_IRQ,
238+ .flags = IORESOURCE_IRQ
239+ }
240+};
241+ 
242+static struct platform_device zilog_device = {
243+ .name = "ip22zilog",
244+ .id = 0,
245+ .num_resources = ARRAY_SIZE(sgi_zilog_resources),
246+ .resource = sgi_zilog_resources,
247+};
248+ 
249+ 
250+static int __init sgi_zilog_devinit(void)
251+{
252+ return platform_device_register(&zilog_device);
253+}
254+ 
255+device_initcall(sgi_zilog_devinit);
March/s390/pci/pci_event.c+121-35
@@ -16,6 +16,7 @@
16#include <asm/sclp.h>16#include <asm/sclp.h>
17 17 
18#include "pci_bus.h"18#include "pci_bus.h"
19+#include "pci_report.h"
19 20 
20/* Content Code Description for PCI Function Error */21/* Content Code Description for PCI Function Error */
21struct zpci_ccdf_err {22struct zpci_ccdf_err {
@@ -53,15 +54,23 @@ static inline bool ers_result_indicates_abort(pci_ers_result_t ers_res)
53 case PCI_ERS_RESULT_CAN_RECOVER:54 case PCI_ERS_RESULT_CAN_RECOVER:
54 case PCI_ERS_RESULT_RECOVERED:55 case PCI_ERS_RESULT_RECOVERED:
55 case PCI_ERS_RESULT_NEED_RESET:56 case PCI_ERS_RESULT_NEED_RESET:
57+ case PCI_ERS_RESULT_NONE:
56 return false;58 return false;
57 default:59 default:
58 return true;60 return true;
59 }61 }
60}62}
61 63 
62-static bool is_passed_through(struct zpci_dev *zdev)64+static bool is_passed_through(struct pci_dev *pdev)
63{65{
64- return zdev->s390_domain;66+ struct zpci_dev *zdev = to_zpci(pdev);
67+ bool ret;
68+ 
69+ mutex_lock(&zdev->kzdev_lock);
70+ ret = !!zdev->kzdev;
71+ mutex_unlock(&zdev->kzdev_lock);
72+ 
73+ return ret;
65}74}
66 75 
67static bool is_driver_supported(struct pci_driver *driver)76static bool is_driver_supported(struct pci_driver *driver)
@@ -70,10 +79,6 @@ static bool is_driver_supported(struct pci_driver *driver)
70 return false;79 return false;
71 if (!driver->err_handler->error_detected)80 if (!driver->err_handler->error_detected)
72 return false;81 return false;
73- if (!driver->err_handler->slot_reset)
74- return false;
75- if (!driver->err_handler->resume)
76- return false;
77 return true;82 return true;
78}83}
79 84 
@@ -83,6 +88,7 @@ static pci_ers_result_t zpci_event_notify_error_detected(struct pci_dev *pdev,
83 pci_ers_result_t ers_res = PCI_ERS_RESULT_DISCONNECT;88 pci_ers_result_t ers_res = PCI_ERS_RESULT_DISCONNECT;
84 89 
85 ers_res = driver->err_handler->error_detected(pdev, pdev->error_state);90 ers_res = driver->err_handler->error_detected(pdev, pdev->error_state);
91+ pci_uevent_ers(pdev, ers_res);
86 if (ers_result_indicates_abort(ers_res))92 if (ers_result_indicates_abort(ers_res))
87 pr_info("%s: Automatic recovery failed after initial reporting\n", pci_name(pdev));93 pr_info("%s: Automatic recovery failed after initial reporting\n", pci_name(pdev));
88 else if (ers_res == PCI_ERS_RESULT_NEED_RESET)94 else if (ers_res == PCI_ERS_RESULT_NEED_RESET)
@@ -110,16 +116,18 @@ static pci_ers_result_t zpci_event_do_error_state_clear(struct pci_dev *pdev,
110 return PCI_ERS_RESULT_NEED_RESET;116 return PCI_ERS_RESULT_NEED_RESET;
111 }117 }
112 118 
113- if (driver->err_handler->mmio_enabled) {119+ if (driver->err_handler->mmio_enabled)
114 ers_res = driver->err_handler->mmio_enabled(pdev);120 ers_res = driver->err_handler->mmio_enabled(pdev);
115- if (ers_result_indicates_abort(ers_res)) {121+ else
116- pr_info("%s: Automatic recovery failed after MMIO re-enable\n",122+ ers_res = PCI_ERS_RESULT_NONE;
117- pci_name(pdev));123+ 
118- return ers_res;124+ if (ers_result_indicates_abort(ers_res)) {
119- } else if (ers_res == PCI_ERS_RESULT_NEED_RESET) {125+ pr_info("%s: Automatic recovery failed after MMIO re-enable\n",
120- pr_debug("%s: Driver needs reset to recover\n", pci_name(pdev));126+ pci_name(pdev));
121- return ers_res;127+ return ers_res;
122- }128+ } else if (ers_res == PCI_ERS_RESULT_NEED_RESET) {
129+ pr_debug("%s: Driver needs reset to recover\n", pci_name(pdev));
130+ return ers_res;
123 }131 }
124 132 
125 pr_debug("%s: Unblocking DMA\n", pci_name(pdev));133 pr_debug("%s: Unblocking DMA\n", pci_name(pdev));
@@ -146,7 +154,12 @@ static pci_ers_result_t zpci_event_do_reset(struct pci_dev *pdev,
146 return ers_res;154 return ers_res;
147 }155 }
148 pdev->error_state = pci_channel_io_normal;156 pdev->error_state = pci_channel_io_normal;
149- ers_res = driver->err_handler->slot_reset(pdev);157+ 
158+ if (driver->err_handler->slot_reset)
159+ ers_res = driver->err_handler->slot_reset(pdev);
160+ else
161+ ers_res = PCI_ERS_RESULT_NONE;
162+ 
150 if (ers_result_indicates_abort(ers_res)) {163 if (ers_result_indicates_abort(ers_res)) {
151 pr_info("%s: Automatic recovery failed after slot reset\n", pci_name(pdev));164 pr_info("%s: Automatic recovery failed after slot reset\n", pci_name(pdev));
152 return ers_res;165 return ers_res;
@@ -166,6 +179,8 @@ static pci_ers_result_t zpci_event_do_reset(struct pci_dev *pdev,
166static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev)179static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev)
167{180{
168 pci_ers_result_t ers_res = PCI_ERS_RESULT_DISCONNECT;181 pci_ers_result_t ers_res = PCI_ERS_RESULT_DISCONNECT;
182+ struct zpci_dev *zdev = to_zpci(pdev);
183+ char *status_str = "success";
169 struct pci_driver *driver;184 struct pci_driver *driver;
170 185 
171 /*186 /*
@@ -173,55 +188,77 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev)
173 * is unbound or probed and that userspace can't access its188 * is unbound or probed and that userspace can't access its
174 * configuration space while we perform recovery.189 * configuration space while we perform recovery.
175 */190 */
176- pci_dev_lock(pdev);191+ device_lock(&pdev->dev);
177 if (pdev->error_state == pci_channel_io_perm_failure) {192 if (pdev->error_state == pci_channel_io_perm_failure) {
178 ers_res = PCI_ERS_RESULT_DISCONNECT;193 ers_res = PCI_ERS_RESULT_DISCONNECT;
179 goto out_unlock;194 goto out_unlock;
180 }195 }
181 pdev->error_state = pci_channel_io_frozen;196 pdev->error_state = pci_channel_io_frozen;
182 197 
183- if (is_passed_through(to_zpci(pdev))) {198+ if (is_passed_through(pdev)) {
184 pr_info("%s: Cannot be recovered in the host because it is a pass-through device\n",199 pr_info("%s: Cannot be recovered in the host because it is a pass-through device\n",
185 pci_name(pdev));200 pci_name(pdev));
201+ status_str = "failed (pass-through)";
186 goto out_unlock;202 goto out_unlock;
187 }203 }
188 204 
189 driver = to_pci_driver(pdev->dev.driver);205 driver = to_pci_driver(pdev->dev.driver);
190 if (!is_driver_supported(driver)) {206 if (!is_driver_supported(driver)) {
191- if (!driver)207+ if (!driver) {
192 pr_info("%s: Cannot be recovered because no driver is bound to the device\n",208 pr_info("%s: Cannot be recovered because no driver is bound to the device\n",
193 pci_name(pdev));209 pci_name(pdev));
194- else210+ status_str = "failed (no driver)";
211+ } else {
195 pr_info("%s: The %s driver bound to the device does not support error recovery\n",212 pr_info("%s: The %s driver bound to the device does not support error recovery\n",
196 pci_name(pdev),213 pci_name(pdev),
197 driver->name);214 driver->name);
215+ status_str = "failed (no driver support)";
216+ }
198 goto out_unlock;217 goto out_unlock;
199 }218 }
200 219 
201 ers_res = zpci_event_notify_error_detected(pdev, driver);220 ers_res = zpci_event_notify_error_detected(pdev, driver);
202- if (ers_result_indicates_abort(ers_res))221+ if (ers_result_indicates_abort(ers_res)) {
222+ status_str = "failed (abort on detection)";
203 goto out_unlock;223 goto out_unlock;
224+ }
204 225 
205- if (ers_res == PCI_ERS_RESULT_CAN_RECOVER) {226+ if (ers_res != PCI_ERS_RESULT_NEED_RESET) {
206 ers_res = zpci_event_do_error_state_clear(pdev, driver);227 ers_res = zpci_event_do_error_state_clear(pdev, driver);
207- if (ers_result_indicates_abort(ers_res))228+ if (ers_result_indicates_abort(ers_res)) {
229+ status_str = "failed (abort on MMIO enable)";
208 goto out_unlock;230 goto out_unlock;
231+ }
209 }232 }
210 233 
211 if (ers_res == PCI_ERS_RESULT_NEED_RESET)234 if (ers_res == PCI_ERS_RESULT_NEED_RESET)
212 ers_res = zpci_event_do_reset(pdev, driver);235 ers_res = zpci_event_do_reset(pdev, driver);
213 236 
237+ /*
238+ * ers_res can be PCI_ERS_RESULT_NONE either because the driver
239+ * decided to return it, indicating that it abstains from voting
240+ * on how to recover, or because it didn't implement the callback.
241+ * Both cases assume, that if there is nothing else causing a
242+ * disconnect, we recovered successfully.
243+ */
244+ if (ers_res == PCI_ERS_RESULT_NONE)
245+ ers_res = PCI_ERS_RESULT_RECOVERED;
246+ 
214 if (ers_res != PCI_ERS_RESULT_RECOVERED) {247 if (ers_res != PCI_ERS_RESULT_RECOVERED) {
248+ pci_uevent_ers(pdev, PCI_ERS_RESULT_DISCONNECT);
215 pr_err("%s: Automatic recovery failed; operator intervention is required\n",249 pr_err("%s: Automatic recovery failed; operator intervention is required\n",
216 pci_name(pdev));250 pci_name(pdev));
251+ status_str = "failed (driver can't recover)";
217 goto out_unlock;252 goto out_unlock;
218 }253 }
219 254 
220 pr_info("%s: The device is ready to resume operations\n", pci_name(pdev));255 pr_info("%s: The device is ready to resume operations\n", pci_name(pdev));
221 if (driver->err_handler->resume)256 if (driver->err_handler->resume)
222 driver->err_handler->resume(pdev);257 driver->err_handler->resume(pdev);
258+ pci_uevent_ers(pdev, PCI_ERS_RESULT_RECOVERED);
223out_unlock:259out_unlock:
224- pci_dev_unlock(pdev);260+ device_unlock(&pdev->dev);
261+ zpci_report_status(zdev, "recovery", status_str);
225 262 
226 return ers_res;263 return ers_res;
227}264}
@@ -243,7 +280,7 @@ static void zpci_event_io_failure(struct pci_dev *pdev, pci_channel_state_t es)
243 * we will inject the error event and let the guest recover the device280 * we will inject the error event and let the guest recover the device
244 * itself.281 * itself.
245 */282 */
246- if (is_passed_through(to_zpci(pdev)))283+ if (is_passed_through(pdev))
247 goto out;284 goto out;
248 driver = to_pci_driver(pdev->dev.driver);285 driver = to_pci_driver(pdev->dev.driver);
249 if (driver && driver->err_handler && driver->err_handler->error_detected)286 if (driver && driver->err_handler && driver->err_handler->error_detected)
@@ -257,6 +294,8 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf)
257 struct zpci_dev *zdev = get_zdev_by_fid(ccdf->fid);294 struct zpci_dev *zdev = get_zdev_by_fid(ccdf->fid);
258 struct pci_dev *pdev = NULL;295 struct pci_dev *pdev = NULL;
259 pci_ers_result_t ers_res;296 pci_ers_result_t ers_res;
297+ u32 fh = 0;
298+ int rc;
260 299 
261 zpci_dbg(3, "err fid:%x, fh:%x, pec:%x\n",300 zpci_dbg(3, "err fid:%x, fh:%x, pec:%x\n",
262 ccdf->fid, ccdf->fh, ccdf->pec);301 ccdf->fid, ccdf->fh, ccdf->pec);
@@ -264,6 +303,16 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf)
264 zpci_err_hex(ccdf, sizeof(*ccdf));303 zpci_err_hex(ccdf, sizeof(*ccdf));
265 304 
266 if (zdev) {305 if (zdev) {
306+ mutex_lock(&zdev->state_lock);
307+ rc = clp_refresh_fh(zdev->fid, &fh);
308+ if (rc)
309+ goto no_pdev;
310+ if (!fh || ccdf->fh != fh) {
311+ /* Ignore events with stale handles */
312+ zpci_dbg(3, "err fid:%x, fh:%x (stale %x)\n",
313+ ccdf->fid, fh, ccdf->fh);
314+ goto no_pdev;
315+ }
267 zpci_update_fh(zdev, ccdf->fh);316 zpci_update_fh(zdev, ccdf->fh);
268 if (zdev->zbus->bus)317 if (zdev->zbus->bus)
269 pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn);318 pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn);
@@ -292,6 +341,8 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf)
292 }341 }
293 pci_dev_put(pdev);342 pci_dev_put(pdev);
294no_pdev:343no_pdev:
344+ if (zdev)
345+ mutex_unlock(&zdev->state_lock);
295 zpci_zdev_put(zdev);346 zpci_zdev_put(zdev);
296}347}
297 348 
@@ -311,13 +362,27 @@ static void zpci_event_hard_deconfigured(struct zpci_dev *zdev, u32 fh)
311 /* Even though the device is already gone we still362 /* Even though the device is already gone we still
312 * need to free zPCI resources as part of the disable.363 * need to free zPCI resources as part of the disable.
313 */364 */
314- if (zdev->dma_table)
315- zpci_dma_exit_device(zdev);
316 if (zdev_enabled(zdev))365 if (zdev_enabled(zdev))
317 zpci_disable_device(zdev);366 zpci_disable_device(zdev);
318 zdev->state = ZPCI_FN_STATE_STANDBY;367 zdev->state = ZPCI_FN_STATE_STANDBY;
319}368}
320 369 
370+static void zpci_event_reappear(struct zpci_dev *zdev)
371+{
372+ lockdep_assert_held(&zdev->state_lock);
373+ /*
374+ * The zdev is in the reserved state. This means that it was presumed to
375+ * go away but there are still undropped references. Now, the platform
376+ * announced its availability again. Bring back the lingering zdev
377+ * to standby. This is safe because we hold a temporary reference
378+ * now so that it won't go away. Account for the re-appearance of the
379+ * underlying device by incrementing the reference count.
380+ */
381+ zdev->state = ZPCI_FN_STATE_STANDBY;
382+ zpci_zdev_get(zdev);
383+ zpci_dbg(1, "rea fid:%x, fh:%x\n", zdev->fid, zdev->fh);
384+}
385+ 
321static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf)386static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf)
322{387{
323 struct zpci_dev *zdev = get_zdev_by_fid(ccdf->fid);388 struct zpci_dev *zdev = get_zdev_by_fid(ccdf->fid);
@@ -326,29 +391,48 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf)
326 391 
327 zpci_dbg(3, "avl fid:%x, fh:%x, pec:%x\n",392 zpci_dbg(3, "avl fid:%x, fh:%x, pec:%x\n",
328 ccdf->fid, ccdf->fh, ccdf->pec);393 ccdf->fid, ccdf->fh, ccdf->pec);
394+ 
395+ if (existing_zdev)
396+ mutex_lock(&zdev->state_lock);
397+ 
329 switch (ccdf->pec) {398 switch (ccdf->pec) {
330 case 0x0301: /* Reserved|Standby -> Configured */399 case 0x0301: /* Reserved|Standby -> Configured */
331 if (!zdev) {400 if (!zdev) {
332 zdev = zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_CONFIGURED);401 zdev = zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_CONFIGURED);
333 if (IS_ERR(zdev))402 if (IS_ERR(zdev))
334 break;403 break;
404+ if (zpci_add_device(zdev)) {
405+ kfree(zdev);
406+ break;
407+ }
335 } else {408 } else {
409+ if (zdev->state == ZPCI_FN_STATE_RESERVED)
410+ zpci_event_reappear(zdev);
336 /* the configuration request may be stale */411 /* the configuration request may be stale */
337- if (zdev->state != ZPCI_FN_STATE_STANDBY)412+ else if (zdev->state != ZPCI_FN_STATE_STANDBY)
338 break;413 break;
339 zdev->state = ZPCI_FN_STATE_CONFIGURED;414 zdev->state = ZPCI_FN_STATE_CONFIGURED;
340 }415 }
341 zpci_scan_configured_device(zdev, ccdf->fh);416 zpci_scan_configured_device(zdev, ccdf->fh);
342 break;417 break;
343 case 0x0302: /* Reserved -> Standby */418 case 0x0302: /* Reserved -> Standby */
344- if (!zdev)419+ if (!zdev) {
345- zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_STANDBY);420+ zdev = zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_STANDBY);
346- else421+ if (IS_ERR(zdev))
422+ break;
423+ if (zpci_add_device(zdev)) {
424+ kfree(zdev);
425+ break;
426+ }
427+ } else {
428+ if (zdev->state == ZPCI_FN_STATE_RESERVED)
429+ zpci_event_reappear(zdev);
347 zpci_update_fh(zdev, ccdf->fh);430 zpci_update_fh(zdev, ccdf->fh);
431+ }
348 break;432 break;
349 case 0x0303: /* Deconfiguration requested */433 case 0x0303: /* Deconfiguration requested */
350 if (zdev) {434 if (zdev) {
351- /* The event may have been queued before we confirgured435+ /* The event may have been queued before we configured
352 * the device.436 * the device.
353 */437 */
354 if (zdev->state != ZPCI_FN_STATE_CONFIGURED)438 if (zdev->state != ZPCI_FN_STATE_CONFIGURED)
@@ -359,7 +443,7 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf)
359 break;443 break;
360 case 0x0304: /* Configured -> Standby|Reserved */444 case 0x0304: /* Configured -> Standby|Reserved */
361 if (zdev) {445 if (zdev) {
362- /* The event may have been queued before we confirgured446+ /* The event may have been queued before we configured
363 * the device.:447 * the device.:
364 */448 */
365 if (zdev->state == ZPCI_FN_STATE_CONFIGURED)449 if (zdev->state == ZPCI_FN_STATE_CONFIGURED)
@@ -373,7 +457,7 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf)
373 break;457 break;
374 case 0x0306: /* 0x308 or 0x302 for multiple devices */458 case 0x0306: /* 0x308 or 0x302 for multiple devices */
375 zpci_remove_reserved_devices();459 zpci_remove_reserved_devices();
376- clp_scan_pci_devices();460+ zpci_scan_devices();
377 break;461 break;
378 case 0x0308: /* Standby -> Reserved */462 case 0x0308: /* Standby -> Reserved */
379 if (!zdev)463 if (!zdev)
@@ -383,8 +467,10 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf)
383 default:467 default:
384 break;468 break;
385 }469 }
386- if (existing_zdev)470+ if (existing_zdev) {
471+ mutex_unlock(&zdev->state_lock);
387 zpci_zdev_put(zdev);472 zpci_zdev_put(zdev);
473+ }
388}474}
389 475 
390void zpci_event_availability(void *data)476void zpci_event_availability(void *data)
March/x86/kernel/cpu/amd.c+292-359
@@ -9,10 +9,12 @@
9#include <linux/sched/clock.h>9#include <linux/sched/clock.h>
10#include <linux/random.h>10#include <linux/random.h>
11#include <linux/topology.h>11#include <linux/topology.h>
12+#include <linux/platform_data/x86/amd-fch.h>
12#include <asm/processor.h>13#include <asm/processor.h>
13#include <asm/apic.h>14#include <asm/apic.h>
14#include <asm/cacheinfo.h>15#include <asm/cacheinfo.h>
15#include <asm/cpu.h>16#include <asm/cpu.h>
17+#include <asm/cpu_device_id.h>
16#include <asm/spec-ctrl.h>18#include <asm/spec-ctrl.h>
17#include <asm/smp.h>19#include <asm/smp.h>
18#include <asm/numa.h>20#include <asm/numa.h>
@@ -20,6 +22,8 @@
20#include <asm/delay.h>22#include <asm/delay.h>
21#include <asm/debugreg.h>23#include <asm/debugreg.h>
22#include <asm/resctrl.h>24#include <asm/resctrl.h>
25+#include <asm/msr.h>
26+#include <asm/sev.h>
23 27 
24#ifdef CONFIG_X86_6428#ifdef CONFIG_X86_64
25# include <asm/mmconfig.h>29# include <asm/mmconfig.h>
@@ -27,81 +31,9 @@
27 31 
28#include "cpu.h"32#include "cpu.h"
29 33 
30-/*34+u16 invlpgb_count_max __ro_after_init = 1;
31- * nodes_per_socket: Stores the number of nodes per socket.
32- * Refer to Fam15h Models 00-0fh BKDG - CPUID Fn8000_001E_ECX
33- * Node Identifiers[10:8]
34- */
35-static u32 nodes_per_socket = 1;
36 35 
37-/*36+static inline int rdmsrq_amd_safe(unsigned msr, u64 *p)
38- * AMD errata checking
39- *
40- * Errata are defined as arrays of ints using the AMD_LEGACY_ERRATUM() or
41- * AMD_OSVW_ERRATUM() macros. The latter is intended for newer errata that
42- * have an OSVW id assigned, which it takes as first argument. Both take a
43- * variable number of family-specific model-stepping ranges created by
44- * AMD_MODEL_RANGE().
45- *
46- * Example:
47- *
48- * const int amd_erratum_319[] =
49- * AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0x4, 0x2),
50- * AMD_MODEL_RANGE(0x10, 0x8, 0x0, 0x8, 0x0),
51- * AMD_MODEL_RANGE(0x10, 0x9, 0x0, 0x9, 0x0));
52- */
53- 
54-#define AMD_LEGACY_ERRATUM(...) { -1, __VA_ARGS__, 0 }
55-#define AMD_OSVW_ERRATUM(osvw_id, ...) { osvw_id, __VA_ARGS__, 0 }
56-#define AMD_MODEL_RANGE(f, m_start, s_start, m_end, s_end) \
57- ((f << 24) | (m_start << 16) | (s_start << 12) | (m_end << 4) | (s_end))
58-#define AMD_MODEL_RANGE_FAMILY(range) (((range) >> 24) & 0xff)
59-#define AMD_MODEL_RANGE_START(range) (((range) >> 12) & 0xfff)
60-#define AMD_MODEL_RANGE_END(range) ((range) & 0xfff)
61- 
62-static const int amd_erratum_400[] =
63- AMD_OSVW_ERRATUM(1, AMD_MODEL_RANGE(0xf, 0x41, 0x2, 0xff, 0xf),
64- AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0xff, 0xf));
65- 
66-static const int amd_erratum_383[] =
67- AMD_OSVW_ERRATUM(3, AMD_MODEL_RANGE(0x10, 0, 0, 0xff, 0xf));
68- 
69-static const int amd_erratum_1485[] =
70- AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x19, 0x10, 0x0, 0x1f, 0xf),
71- AMD_MODEL_RANGE(0x19, 0x60, 0x0, 0xaf, 0xf));
72- 
73-static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum)
74-{
75- int osvw_id = *erratum++;
76- u32 range;
77- u32 ms;
78- 
79- if (osvw_id >= 0 && osvw_id < 65536 &&
80- cpu_has(cpu, X86_FEATURE_OSVW)) {
81- u64 osvw_len;
82- 
83- rdmsrl(MSR_AMD64_OSVW_ID_LENGTH, osvw_len);
84- if (osvw_id < osvw_len) {
85- u64 osvw_bits;
86- 
87- rdmsrl(MSR_AMD64_OSVW_STATUS + (osvw_id >> 6),
88- osvw_bits);
89- return osvw_bits & (1ULL << (osvw_id & 0x3f));
90- }
91- }
92- 
93- /* OSVW unavailable or ID unknown, match family-model-stepping range */
94- ms = (cpu->x86_model << 4) | cpu->x86_stepping;
95- while ((range = *erratum++))
96- if ((cpu->x86 == AMD_MODEL_RANGE_FAMILY(range)) &&
97- (ms >= AMD_MODEL_RANGE_START(range)) &&
98- (ms <= AMD_MODEL_RANGE_END(range)))
99- return true;
100- 
101- return false;
102-}
103- 
104-static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
105{37{
106 u32 gprs[8] = { 0 };38 u32 gprs[8] = { 0 };
107 int err;39 int err;
@@ -119,7 +51,7 @@ static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
119 return err;51 return err;
120}52}
121 53 
122-static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)54+static inline int wrmsrq_amd_safe(unsigned msr, u64 val)
123{55{
124 u32 gprs[8] = { 0 };56 u32 gprs[8] = { 0 };
125 57 
@@ -367,111 +299,16 @@ static int nearby_node(int apicid)
367}299}
368#endif300#endif
369 301 
370-/*
371- * Fix up cpu_core_id for pre-F17h systems to be in the
372- * [0 .. cores_per_node - 1] range. Not really needed but
373- * kept so as not to break existing setups.
374- */
375-static void legacy_fixup_core_id(struct cpuinfo_x86 *c)
376-{
377- u32 cus_per_node;
378- 
379- if (c->x86 >= 0x17)
380- return;
381- 
382- cus_per_node = c->x86_max_cores / nodes_per_socket;
383- c->cpu_core_id %= cus_per_node;
384-}
385- 
386-/*
387- * Fixup core topology information for
388- * (1) AMD multi-node processors
389- * Assumption: Number of cores in each internal node is the same.
390- * (2) AMD processors supporting compute units
391- */
392-static void amd_get_topology(struct cpuinfo_x86 *c)
393-{
394- int cpu = smp_processor_id();
395- 
396- /* get information required for multi-node processors */
397- if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
398- int err;
399- u32 eax, ebx, ecx, edx;
400- 
401- cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
402- 
403- c->cpu_die_id = ecx & 0xff;
404- 
405- if (c->x86 == 0x15)
406- c->cu_id = ebx & 0xff;
407- 
408- if (c->x86 >= 0x17) {
409- c->cpu_core_id = ebx & 0xff;
410- 
411- if (smp_num_siblings > 1)
412- c->x86_max_cores /= smp_num_siblings;
413- }
414- 
415- /*
416- * In case leaf B is available, use it to derive
417- * topology information.
418- */
419- err = detect_extended_topology(c);
420- if (!err)
421- c->x86_coreid_bits = get_count_order(c->x86_max_cores);
422- 
423- cacheinfo_amd_init_llc_id(c, cpu);
424- 
425- } else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) {
426- u64 value;
427- 
428- rdmsrl(MSR_FAM10H_NODE_ID, value);
429- c->cpu_die_id = value & 7;
430- 
431- per_cpu(cpu_llc_id, cpu) = c->cpu_die_id;
432- } else
433- return;
434- 
435- if (nodes_per_socket > 1) {
436- set_cpu_cap(c, X86_FEATURE_AMD_DCM);
437- legacy_fixup_core_id(c);
438- }
439-}
440- 
441-/*
442- * On a AMD dual core setup the lower bits of the APIC id distinguish the cores.
443- * Assumes number of cores is a power of two.
444- */
445-static void amd_detect_cmp(struct cpuinfo_x86 *c)
446-{
447- unsigned bits;
448- int cpu = smp_processor_id();
449- 
450- bits = c->x86_coreid_bits;
451- /* Low order bits define the core id (index of core in socket) */
452- c->cpu_core_id = c->initial_apicid & ((1 << bits)-1);
453- /* Convert the initial APIC ID into the socket ID */
454- c->phys_proc_id = c->initial_apicid >> bits;
455- /* use socket ID also for last level cache */
456- per_cpu(cpu_llc_id, cpu) = c->cpu_die_id = c->phys_proc_id;
457-}
458- 
459-u32 amd_get_nodes_per_socket(void)
460-{
461- return nodes_per_socket;
462-}
463-EXPORT_SYMBOL_GPL(amd_get_nodes_per_socket);
464- 
465static void srat_detect_node(struct cpuinfo_x86 *c)302static void srat_detect_node(struct cpuinfo_x86 *c)
466{303{
467#ifdef CONFIG_NUMA304#ifdef CONFIG_NUMA
468 int cpu = smp_processor_id();305 int cpu = smp_processor_id();
469 int node;306 int node;
470- unsigned apicid = c->apicid;307+ unsigned apicid = c->topo.apicid;
471 308 
472 node = numa_cpu_node(cpu);309 node = numa_cpu_node(cpu);
473 if (node == NUMA_NO_NODE)310 if (node == NUMA_NO_NODE)
474- node = get_llc_id(cpu);311+ node = per_cpu_llc_id(cpu);
475 312 
476 /*313 /*
477 * On multi-fabric platform (e.g. Numascale NumaChip) a314 * On multi-fabric platform (e.g. Numascale NumaChip) a
@@ -501,7 +338,7 @@ static void srat_detect_node(struct cpuinfo_x86 *c)
501 * through CPU mapping may alter the outcome, directly338 * through CPU mapping may alter the outcome, directly
502 * access __apicid_to_node[].339 * access __apicid_to_node[].
503 */340 */
504- int ht_nodeid = c->initial_apicid;341+ int ht_nodeid = c->topo.initial_apicid;
505 342 
506 if (__apicid_to_node[ht_nodeid] != NUMA_NO_NODE)343 if (__apicid_to_node[ht_nodeid] != NUMA_NO_NODE)
507 node = __apicid_to_node[ht_nodeid];344 node = __apicid_to_node[ht_nodeid];
@@ -513,76 +350,56 @@ static void srat_detect_node(struct cpuinfo_x86 *c)
513#endif350#endif
514}351}
515 352 
516-static void early_init_amd_mc(struct cpuinfo_x86 *c)353+static void bsp_determine_snp(struct cpuinfo_x86 *c)
517{354{
518-#ifdef CONFIG_SMP355+#ifdef CONFIG_ARCH_HAS_CC_PLATFORM
519- unsigned bits, ecx;356+ cc_vendor = CC_VENDOR_AMD;
520 357 
521- /* Multi core CPU? */358+ if (cpu_has(c, X86_FEATURE_SEV_SNP)) {
522- if (c->extended_cpuid_level < 0x80000008)359+ /*
523- return;360+ * RMP table entry format is not architectural and is defined by the
524- 361+ * per-processor PPR. Restrict SNP support on the known CPU models
525- ecx = cpuid_ecx(0x80000008);362+ * for which the RMP table entry format is currently defined or for
526- 363+ * processors which support the architecturally defined RMPREAD
527- c->x86_max_cores = (ecx & 0xff) + 1;364+ * instruction.
528- 365+ */
529- /* CPU telling us the core id bits shift? */366+ if (!cpu_has(c, X86_FEATURE_HYPERVISOR) &&
530- bits = (ecx >> 12) & 0xF;367+ (cpu_feature_enabled(X86_FEATURE_ZEN3) ||
531- 368+ cpu_feature_enabled(X86_FEATURE_ZEN4) ||
532- /* Otherwise recompute */369+ cpu_feature_enabled(X86_FEATURE_RMPREAD)) &&
533- if (bits == 0) {370+ snp_probe_rmptable_info()) {
534- while ((1 << bits) < c->x86_max_cores)371+ cc_platform_set(CC_ATTR_HOST_SEV_SNP);
535- bits++;372+ } else {
373+ setup_clear_cpu_cap(X86_FEATURE_SEV_SNP);
374+ cc_platform_clear(CC_ATTR_HOST_SEV_SNP);
375+ }
536 }376 }
537- 
538- c->x86_coreid_bits = bits;
539#endif377#endif
540}378}
541 379 
542-static bool amd_check_tsa_microcode(void)380+#define ZEN_MODEL_STEP_UCODE(fam, model, step, ucode) \
543-{381+ X86_MATCH_VFM_STEPS(VFM_MAKE(X86_VENDOR_AMD, fam, model), \
544- struct cpuinfo_x86 *c = &boot_cpu_data;382+ step, step, ucode)
545- union zen_patch_rev p;
546- u32 min_rev = 0;
547 383 
548- p.ext_fam = c->x86 - 0xf;384+static const struct x86_cpu_id amd_tsa_microcode[] = {
549- p.model = c->x86_model;385+ ZEN_MODEL_STEP_UCODE(0x19, 0x01, 0x1, 0x0a0011d7),
550- p.ext_model = c->x86_model >> 4;386+ ZEN_MODEL_STEP_UCODE(0x19, 0x01, 0x2, 0x0a00123b),
551- p.stepping = c->x86_stepping;387+ ZEN_MODEL_STEP_UCODE(0x19, 0x08, 0x2, 0x0a00820d),
552- /* reserved bits are expected to be 0 in test below */388+ ZEN_MODEL_STEP_UCODE(0x19, 0x11, 0x1, 0x0a10114c),
553- p.__reserved = 0;389+ ZEN_MODEL_STEP_UCODE(0x19, 0x11, 0x2, 0x0a10124c),
554- 390+ ZEN_MODEL_STEP_UCODE(0x19, 0x18, 0x1, 0x0a108109),
555- if (cpu_has(c, X86_FEATURE_ZEN3) ||391+ ZEN_MODEL_STEP_UCODE(0x19, 0x21, 0x0, 0x0a20102e),
556- cpu_has(c, X86_FEATURE_ZEN4)) {392+ ZEN_MODEL_STEP_UCODE(0x19, 0x21, 0x2, 0x0a201211),
557- switch (p.ucode_rev >> 8) {393+ ZEN_MODEL_STEP_UCODE(0x19, 0x44, 0x1, 0x0a404108),
558- case 0xa0011: min_rev = 0x0a0011d7; break;394+ ZEN_MODEL_STEP_UCODE(0x19, 0x50, 0x0, 0x0a500012),
559- case 0xa0012: min_rev = 0x0a00123b; break;395+ ZEN_MODEL_STEP_UCODE(0x19, 0x61, 0x2, 0x0a60120a),
560- case 0xa0082: min_rev = 0x0a00820d; break;396+ ZEN_MODEL_STEP_UCODE(0x19, 0x74, 0x1, 0x0a704108),
561- case 0xa1011: min_rev = 0x0a10114c; break;397+ ZEN_MODEL_STEP_UCODE(0x19, 0x75, 0x2, 0x0a705208),
562- case 0xa1012: min_rev = 0x0a10124c; break;398+ ZEN_MODEL_STEP_UCODE(0x19, 0x78, 0x0, 0x0a708008),
563- case 0xa1081: min_rev = 0x0a108109; break;399+ ZEN_MODEL_STEP_UCODE(0x19, 0x7c, 0x0, 0x0a70c008),
564- case 0xa2010: min_rev = 0x0a20102e; break;400+ ZEN_MODEL_STEP_UCODE(0x19, 0xa0, 0x2, 0x0aa00216),
565- case 0xa2012: min_rev = 0x0a201211; break;401+ {},
566- case 0xa4041: min_rev = 0x0a404108; break;402+};
567- case 0xa5000: min_rev = 0x0a500012; break;
568- case 0xa6012: min_rev = 0x0a60120a; break;
569- case 0xa7041: min_rev = 0x0a704108; break;
570- case 0xa7052: min_rev = 0x0a705208; break;
571- case 0xa7080: min_rev = 0x0a708008; break;
572- case 0xa70c0: min_rev = 0x0a70c008; break;
573- case 0xaa002: min_rev = 0x0aa00216; break;
574- default:
575- pr_debug("%s: ucode_rev: 0x%x, current revision: 0x%x\n",
576- __func__, p.ucode_rev, c->microcode);
577- return false;
578- }
579- }
580- 
581- if (!min_rev)
582- return false;
583- 
584- return c->microcode >= min_rev;
585-}
586 403 
587static void tsa_init(struct cpuinfo_x86 *c)404static void tsa_init(struct cpuinfo_x86 *c)
588{405{
@@ -591,8 +408,10 @@ static void tsa_init(struct cpuinfo_x86 *c)
591 408 
592 if (cpu_has(c, X86_FEATURE_ZEN3) ||409 if (cpu_has(c, X86_FEATURE_ZEN3) ||
593 cpu_has(c, X86_FEATURE_ZEN4)) {410 cpu_has(c, X86_FEATURE_ZEN4)) {
594- if (amd_check_tsa_microcode())411+ if (x86_match_min_microcode_rev(amd_tsa_microcode))
595 setup_force_cpu_cap(X86_FEATURE_VERW_CLEAR);412 setup_force_cpu_cap(X86_FEATURE_VERW_CLEAR);
413+ else
414+ pr_debug("%s: current revision: 0x%x\n", __func__, c->microcode);
596 } else {415 } else {
597 setup_force_cpu_cap(X86_FEATURE_TSA_SQ_NO);416 setup_force_cpu_cap(X86_FEATURE_TSA_SQ_NO);
598 setup_force_cpu_cap(X86_FEATURE_TSA_L1_NO);417 setup_force_cpu_cap(X86_FEATURE_TSA_L1_NO);
@@ -607,7 +426,7 @@ static void bsp_init_amd(struct cpuinfo_x86 *c)
607 (c->x86 == 0x10 && c->x86_model >= 0x2)) {426 (c->x86 == 0x10 && c->x86_model >= 0x2)) {
608 u64 val;427 u64 val;
609 428 
610- rdmsrl(MSR_K7_HWCR, val);429+ rdmsrq(MSR_K7_HWCR, val);
611 if (!(val & BIT(24)))430 if (!(val & BIT(24)))
612 pr_warn(FW_BUG "TSC doesn't count with P0 frequency!\n");431 pr_warn(FW_BUG "TSC doesn't count with P0 frequency!\n");
613 }432 }
@@ -631,18 +450,6 @@ static void bsp_init_amd(struct cpuinfo_x86 *c)
631 if (cpu_has(c, X86_FEATURE_MWAITX))450 if (cpu_has(c, X86_FEATURE_MWAITX))
632 use_mwaitx_delay();451 use_mwaitx_delay();
633 452 
634- if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
635- u32 ecx;
636- 
637- ecx = cpuid_ecx(0x8000001e);
638- __max_die_per_package = nodes_per_socket = ((ecx >> 8) & 7) + 1;
639- } else if (boot_cpu_has(X86_FEATURE_NODEID_MSR)) {
640- u64 value;
641- 
642- rdmsrl(MSR_FAM10H_NODE_ID, value);
643- __max_die_per_package = nodes_per_socket = ((value >> 3) & 7) + 1;
644- }
645- 
646 if (!boot_cpu_has(X86_FEATURE_AMD_SSBD) &&453 if (!boot_cpu_has(X86_FEATURE_AMD_SSBD) &&
647 !boot_cpu_has(X86_FEATURE_VIRT_SSBD) &&454 !boot_cpu_has(X86_FEATURE_VIRT_SSBD) &&
648 c->x86 >= 0x15 && c->x86 <= 0x17) {455 c->x86 >= 0x15 && c->x86 <= 0x17) {
@@ -658,7 +465,7 @@ static void bsp_init_amd(struct cpuinfo_x86 *c)
658 * Try to cache the base value so further operations can465 * Try to cache the base value so further operations can
659 * avoid RMW. If that faults, do not enable SSBD.466 * avoid RMW. If that faults, do not enable SSBD.
660 */467 */
661- if (!rdmsrl_safe(MSR_AMD64_LS_CFG, &x86_amd_ls_cfg_base)) {468+ if (!rdmsrq_safe(MSR_AMD64_LS_CFG, &x86_amd_ls_cfg_base)) {
662 setup_force_cpu_cap(X86_FEATURE_LS_CFG_SSBD);469 setup_force_cpu_cap(X86_FEATURE_LS_CFG_SSBD);
663 setup_force_cpu_cap(X86_FEATURE_SSBD);470 setup_force_cpu_cap(X86_FEATURE_SSBD);
664 x86_amd_ls_cfg_ssbd_mask = 1ULL << bit;471 x86_amd_ls_cfg_ssbd_mask = 1ULL << bit;
@@ -669,7 +476,7 @@ static void bsp_init_amd(struct cpuinfo_x86 *c)
669 476 
670 /* Figure out Zen generations: */477 /* Figure out Zen generations: */
671 switch (c->x86) {478 switch (c->x86) {
672- case 0x17: {479+ case 0x17:
673 switch (c->x86_model) {480 switch (c->x86_model) {
674 case 0x00 ... 0x2f:481 case 0x00 ... 0x2f:
675 case 0x50 ... 0x5f:482 case 0x50 ... 0x5f:
@@ -685,8 +492,8 @@ static void bsp_init_amd(struct cpuinfo_x86 *c)
685 goto warn;492 goto warn;
686 }493 }
687 break;494 break;
688- }495+ 
689- case 0x19: {496+ case 0x19:
690 switch (c->x86_model) {497 switch (c->x86_model) {
691 case 0x00 ... 0x0f:498 case 0x00 ... 0x0f:
692 case 0x20 ... 0x5f:499 case 0x20 ... 0x5f:
@@ -700,14 +507,34 @@ static void bsp_init_amd(struct cpuinfo_x86 *c)
700 goto warn;507 goto warn;
701 }508 }
702 break;509 break;
703- }510+ 
511+ case 0x1a:
512+ switch (c->x86_model) {
513+ case 0x00 ... 0x2f:
514+ case 0x40 ... 0x4f:
515+ case 0x60 ... 0x7f:
516+ setup_force_cpu_cap(X86_FEATURE_ZEN5);
517+ break;
518+ case 0x50 ... 0x5f:
519+ case 0x90 ... 0xaf:
520+ case 0xc0 ... 0xcf:
521+ setup_force_cpu_cap(X86_FEATURE_ZEN6);
522+ break;
523+ default:
524+ goto warn;
525+ }
526+ break;
527+ 
704 default:528 default:
705 break;529 break;
706 }530 }
707 531 
708- 532+ bsp_determine_snp(c);
709 tsa_init(c);533 tsa_init(c);
710 534 
535+ if (cpu_has(c, X86_FEATURE_GP_ON_USER_CPUID))
536+ setup_force_cpu_cap(X86_FEATURE_CPUID_FAULT);
537+ 
711 return;538 return;
712 539 
713warn:540warn:
@@ -718,6 +545,23 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c)
718{545{
719 u64 msr;546 u64 msr;
720 547 
548+ /*
549+ * Mark using WBINVD is needed during kexec on processors that
550+ * support SME. This provides support for performing a successful
551+ * kexec when going from SME inactive to SME active (or vice-versa).
552+ *
553+ * The cache must be cleared so that if there are entries with the
554+ * same physical address, both with and without the encryption bit,
555+ * they don't race each other when flushed and potentially end up
556+ * with the wrong entry being committed to memory.
557+ *
558+ * Test the CPUID bit directly because with mem_encrypt=off the
559+ * BSP will clear the X86_FEATURE_SME bit and the APs will not
560+ * see it set after that.
561+ */
562+ if (c->extended_cpuid_level >= 0x8000001f && (cpuid_eax(0x8000001f) & BIT(0)))
563+ __this_cpu_write(cache_state_incoherent, true);
564+ 
721 /*565 /*
722 * BIOS support is required for SME and SEV.566 * BIOS support is required for SME and SEV.
723 * For SME: If BIOS has enabled SME then adjust x86_phys_bits by567 * For SME: If BIOS has enabled SME then adjust x86_phys_bits by
@@ -726,15 +570,15 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c)
726 * SME feature (set in scattered.c).570 * SME feature (set in scattered.c).
727 * If the kernel has not enabled SME via any means then571 * If the kernel has not enabled SME via any means then
728 * don't advertise the SME feature.572 * don't advertise the SME feature.
729- * For SEV: If BIOS has not enabled SEV then don't advertise the573+ * For SEV: If BIOS has not enabled SEV then don't advertise SEV and
730- * SEV and SEV_ES feature (set in scattered.c).574+ * any additional functionality based on it.
731 *575 *
732 * In all cases, since support for SME and SEV requires long mode,576 * In all cases, since support for SME and SEV requires long mode,
733 * don't advertise the feature under CONFIG_X86_32.577 * don't advertise the feature under CONFIG_X86_32.
734 */578 */
735 if (cpu_has(c, X86_FEATURE_SME) || cpu_has(c, X86_FEATURE_SEV)) {579 if (cpu_has(c, X86_FEATURE_SME) || cpu_has(c, X86_FEATURE_SEV)) {
736 /* Check if memory encryption is enabled */580 /* Check if memory encryption is enabled */
737- rdmsrl(MSR_AMD64_SYSCFG, msr);581+ rdmsrq(MSR_AMD64_SYSCFG, msr);
738 if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))582 if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
739 goto clear_all;583 goto clear_all;
740 584 
@@ -751,7 +595,7 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c)
751 if (!sme_me_mask)595 if (!sme_me_mask)
752 setup_clear_cpu_cap(X86_FEATURE_SME);596 setup_clear_cpu_cap(X86_FEATURE_SME);
753 597 
754- rdmsrl(MSR_K7_HWCR, msr);598+ rdmsrq(MSR_K7_HWCR, msr);
755 if (!(msr & MSR_K7_HWCR_SMMLOCK))599 if (!(msr & MSR_K7_HWCR_SMMLOCK))
756 goto clear_sev;600 goto clear_sev;
757 601 
@@ -762,16 +606,14 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c)
762clear_sev:606clear_sev:
763 setup_clear_cpu_cap(X86_FEATURE_SEV);607 setup_clear_cpu_cap(X86_FEATURE_SEV);
764 setup_clear_cpu_cap(X86_FEATURE_SEV_ES);608 setup_clear_cpu_cap(X86_FEATURE_SEV_ES);
609+ setup_clear_cpu_cap(X86_FEATURE_SEV_SNP);
765 }610 }
766}611}
767 612 
768static void early_init_amd(struct cpuinfo_x86 *c)613static void early_init_amd(struct cpuinfo_x86 *c)
769{614{
770- u64 value;
771 u32 dummy;615 u32 dummy;
772 616 
773- early_init_amd_mc(c);
774- 
775 if (c->x86 >= 0xf)617 if (c->x86 >= 0xf)
776 set_cpu_cap(c, X86_FEATURE_K8);618 set_cpu_cap(c, X86_FEATURE_K8);
777 619 
@@ -835,38 +677,12 @@ static void early_init_amd(struct cpuinfo_x86 *c)
835 if (c->x86 == 0x16 && c->x86_model <= 0xf)677 if (c->x86 == 0x16 && c->x86_model <= 0xf)
836 msr_set_bit(MSR_AMD64_LS_CFG, 15);678 msr_set_bit(MSR_AMD64_LS_CFG, 15);
837 679 
838- /*
839- * Check whether the machine is affected by erratum 400. This is
840- * used to select the proper idle routine and to enable the check
841- * whether the machine is affected in arch_post_acpi_init(), which
842- * sets the X86_BUG_AMD_APIC_C1E bug depending on the MSR check.
843- */
844- if (cpu_has_amd_erratum(c, amd_erratum_400))
845- set_cpu_bug(c, X86_BUG_AMD_E400);
846- 
847 early_detect_mem_encrypt(c);680 early_detect_mem_encrypt(c);
848 681 
849- /* Re-enable TopologyExtensions if switched off by BIOS */
850- if (c->x86 == 0x15 &&
851- (c->x86_model >= 0x10 && c->x86_model <= 0x6f) &&
852- !cpu_has(c, X86_FEATURE_TOPOEXT)) {
853- 
854- if (msr_set_bit(0xc0011005, 54) > 0) {
855- rdmsrl(0xc0011005, value);
856- if (value & BIT_64(54)) {
857- set_cpu_cap(c, X86_FEATURE_TOPOEXT);
858- pr_info_once(FW_INFO "CPU: Re-enabling disabled Topology Extensions Support.\n");
859- }
860- }
861- }
862- 
863- if (cpu_has(c, X86_FEATURE_TOPOEXT))
864- smp_num_siblings = ((cpuid_ebx(0x8000001e) >> 8) & 0xff) + 1;
865- 
866 if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && !cpu_has(c, X86_FEATURE_IBPB_BRTYPE)) {682 if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && !cpu_has(c, X86_FEATURE_IBPB_BRTYPE)) {
867 if (c->x86 == 0x17 && boot_cpu_has(X86_FEATURE_AMD_IBPB))683 if (c->x86 == 0x17 && boot_cpu_has(X86_FEATURE_AMD_IBPB))
868 setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE);684 setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE);
869- else if (c->x86 >= 0x19 && !wrmsrl_safe(MSR_IA32_PRED_CMD, PRED_CMD_SBPB)) {685+ else if (c->x86 >= 0x19 && !wrmsrq_safe(MSR_IA32_PRED_CMD, PRED_CMD_SBPB)) {
870 setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE);686 setup_force_cpu_cap(X86_FEATURE_IBPB_BRTYPE);
871 setup_force_cpu_cap(X86_FEATURE_SBPB);687 setup_force_cpu_cap(X86_FEATURE_SBPB);
872 }688 }
@@ -890,14 +706,14 @@ static void init_amd_k8(struct cpuinfo_x86 *c)
890 */706 */
891 if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM) && !cpu_has(c, X86_FEATURE_HYPERVISOR)) {707 if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM) && !cpu_has(c, X86_FEATURE_HYPERVISOR)) {
892 clear_cpu_cap(c, X86_FEATURE_LAHF_LM);708 clear_cpu_cap(c, X86_FEATURE_LAHF_LM);
893- if (!rdmsrl_amd_safe(0xc001100d, &value)) {709+ if (!rdmsrq_amd_safe(0xc001100d, &value)) {
894 value &= ~BIT_64(32);710 value &= ~BIT_64(32);
895- wrmsrl_amd_safe(0xc001100d, value);711+ wrmsrq_amd_safe(0xc001100d, value);
896 }712 }
897 }713 }
898 714 
899 if (!c->x86_model_id[0])715 if (!c->x86_model_id[0])
900- strcpy(c->x86_model_id, "Hammer");716+ strscpy(c->x86_model_id, "Hammer");
901 717 
902#ifdef CONFIG_SMP718#ifdef CONFIG_SMP
903 /*719 /*
@@ -910,6 +726,16 @@ static void init_amd_k8(struct cpuinfo_x86 *c)
910 msr_set_bit(MSR_K7_HWCR, 6);726 msr_set_bit(MSR_K7_HWCR, 6);
911#endif727#endif
912 set_cpu_bug(c, X86_BUG_SWAPGS_FENCE);728 set_cpu_bug(c, X86_BUG_SWAPGS_FENCE);
729+ 
730+ /*
731+ * Check models and steppings affected by erratum 400. This is
732+ * used to select the proper idle routine and to enable the
733+ * check whether the machine is affected in arch_post_acpi_subsys_init()
734+ * which sets the X86_BUG_AMD_APIC_C1E bug depending on the MSR check.
735+ */
736+ if (c->x86_model > 0x41 ||
737+ (c->x86_model == 0x41 && c->x86_stepping >= 0x2))
738+ setup_force_cpu_bug(X86_BUG_AMD_E400);
913}739}
914 740 
915static void init_amd_gh(struct cpuinfo_x86 *c)741static void init_amd_gh(struct cpuinfo_x86 *c)
@@ -943,8 +769,17 @@ static void init_amd_gh(struct cpuinfo_x86 *c)
943 */769 */
944 msr_clear_bit(MSR_AMD64_BU_CFG2, 24);770 msr_clear_bit(MSR_AMD64_BU_CFG2, 24);
945 771 
946- if (cpu_has_amd_erratum(c, amd_erratum_383))772+ set_cpu_bug(c, X86_BUG_AMD_TLB_MMATCH);
947- set_cpu_bug(c, X86_BUG_AMD_TLB_MMATCH);773+ 
774+ /*
775+ * Check models and steppings affected by erratum 400. This is
776+ * used to select the proper idle routine and to enable the
777+ * check whether the machine is affected in arch_post_acpi_subsys_init()
778+ * which sets the X86_BUG_AMD_APIC_C1E bug depending on the MSR check.
779+ */
780+ if (c->x86_model > 0x2 ||
781+ (c->x86_model == 0x2 && c->x86_stepping >= 0x1))
782+ setup_force_cpu_bug(X86_BUG_AMD_E400);
948}783}
949 784 
950static void init_amd_ln(struct cpuinfo_x86 *c)785static void init_amd_ln(struct cpuinfo_x86 *c)
@@ -1023,9 +858,9 @@ static void init_amd_bd(struct cpuinfo_x86 *c)
1023 * Disable it on the affected CPUs.858 * Disable it on the affected CPUs.
1024 */859 */
1025 if ((c->x86_model >= 0x02) && (c->x86_model < 0x20)) {860 if ((c->x86_model >= 0x02) && (c->x86_model < 0x20)) {
1026- if (!rdmsrl_safe(MSR_F15H_IC_CFG, &value) && !(value & 0x1E)) {861+ if (!rdmsrq_safe(MSR_F15H_IC_CFG, &value) && !(value & 0x1E)) {
1027 value |= 0x1E;862 value |= 0x1E;
1028- wrmsrl_safe(MSR_F15H_IC_CFG, value);863+ wrmsrq_safe(MSR_F15H_IC_CFG, value);
1029 }864 }
1030 }865 }
1031 866 
@@ -1037,6 +872,12 @@ static void init_amd_bd(struct cpuinfo_x86 *c)
1037 clear_rdrand_cpuid_bit(c);872 clear_rdrand_cpuid_bit(c);
1038}873}
1039 874 
875+static const struct x86_cpu_id erratum_1386_microcode[] = {
876+ X86_MATCH_VFM_STEPS(VFM_MAKE(X86_VENDOR_AMD, 0x17, 0x01), 0x2, 0x2, 0x0800126e),
877+ X86_MATCH_VFM_STEPS(VFM_MAKE(X86_VENDOR_AMD, 0x17, 0x31), 0x0, 0x0, 0x08301052),
878+ {}
879+};
880+ 
1040static void fix_erratum_1386(struct cpuinfo_x86 *c)881static void fix_erratum_1386(struct cpuinfo_x86 *c)
1041{882{
1042 /*883 /*
@@ -1046,13 +887,19 @@ static void fix_erratum_1386(struct cpuinfo_x86 *c)
1046 *887 *
1047 * Affected parts all have no supervisor XSAVE states, meaning that888 * Affected parts all have no supervisor XSAVE states, meaning that
1048 * the XSAVEC instruction (which works fine) is equivalent.889 * the XSAVEC instruction (which works fine) is equivalent.
890+ *
891+ * Clear the feature flag only on microcode revisions which
892+ * don't have the fix.
1049 */893 */
894+ if (x86_match_min_microcode_rev(erratum_1386_microcode))
895+ return;
896+ 
1050 clear_cpu_cap(c, X86_FEATURE_XSAVES);897 clear_cpu_cap(c, X86_FEATURE_XSAVES);
1051}898}
1052 899 
1053void init_spectral_chicken(struct cpuinfo_x86 *c)900void init_spectral_chicken(struct cpuinfo_x86 *c)
1054{901{
1055-#ifdef CONFIG_CPU_UNRET_ENTRY902+#ifdef CONFIG_MITIGATION_UNRET_ENTRY
1056 u64 value;903 u64 value;
1057 904 
1058 /*905 /*
@@ -1060,19 +907,17 @@ void init_spectral_chicken(struct cpuinfo_x86 *c)
1060 *907 *
1061 * This suppresses speculation from the middle of a basic block, i.e. it908 * This suppresses speculation from the middle of a basic block, i.e. it
1062 * suppresses non-branch predictions.909 * suppresses non-branch predictions.
1063- *
1064- * We use STIBP as a heuristic to filter out Zen2 from the rest of F17H
1065 */910 */
1066- if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && cpu_has(c, X86_FEATURE_AMD_STIBP)) {911+ if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) {
1067- if (!rdmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, &value)) {912+ if (!rdmsrq_safe(MSR_ZEN2_SPECTRAL_CHICKEN, &value)) {
1068 value |= MSR_ZEN2_SPECTRAL_CHICKEN_BIT;913 value |= MSR_ZEN2_SPECTRAL_CHICKEN_BIT;
1069- wrmsrl_safe(MSR_ZEN2_SPECTRAL_CHICKEN, value);914+ wrmsrq_safe(MSR_ZEN2_SPECTRAL_CHICKEN, value);
1070 }915 }
1071 }916 }
1072#endif917#endif
1073}918}
1074 919 
1075-static void init_amd_zn(struct cpuinfo_x86 *c)920+static void init_amd_zen_common(void)
1076{921{
1077 setup_force_cpu_cap(X86_FEATURE_ZEN);922 setup_force_cpu_cap(X86_FEATURE_ZEN);
1078#ifdef CONFIG_NUMA923#ifdef CONFIG_NUMA
@@ -1090,14 +935,6 @@ static void init_amd_zen1(struct cpuinfo_x86 *c)
1090 /* Erratum 1076: CPB feature bit not being set in CPUID. */935 /* Erratum 1076: CPB feature bit not being set in CPUID. */
1091 if (!cpu_has(c, X86_FEATURE_CPB))936 if (!cpu_has(c, X86_FEATURE_CPB))
1092 set_cpu_cap(c, X86_FEATURE_CPB);937 set_cpu_cap(c, X86_FEATURE_CPB);
1093- 
1094- /*
1095- * Zen3 (Fam19 model < 0x10) parts are not susceptible to
1096- * Branch Type Confusion, but predate the allocation of the
1097- * BTC_NO bit.
1098- */
1099- if (c->x86 == 0x19 && !cpu_has(c, X86_FEATURE_BTC_NO))
1100- set_cpu_cap(c, X86_FEATURE_BTC_NO);
1101 }938 }
1102 939 
1103 pr_notice_once("AMD Zen1 DIV0 bug detected. Disable SMT for full protection.\n");940 pr_notice_once("AMD Zen1 DIV0 bug detected. Disable SMT for full protection.\n");
@@ -1127,7 +964,6 @@ static bool cpu_has_zenbleed_microcode(void)
1127 964 
1128 default:965 default:
1129 return false;966 return false;
1130- break;
1131 }967 }
1132 968 
1133 if (boot_cpu_data.microcode < good_rev)969 if (boot_cpu_data.microcode < good_rev)
@@ -1154,6 +990,7 @@ static void zen2_zenbleed_check(struct cpuinfo_x86 *c)
1154 990 
1155static void init_amd_zen2(struct cpuinfo_x86 *c)991static void init_amd_zen2(struct cpuinfo_x86 *c)
1156{992{
993+ init_spectral_chicken(c);
1157 fix_erratum_1386(c);994 fix_erratum_1386(c);
1158 zen2_zenbleed_check(c);995 zen2_zenbleed_check(c);
1159 996 
@@ -1163,18 +1000,59 @@ static void init_amd_zen2(struct cpuinfo_x86 *c)
1163 msr_clear_bit(MSR_AMD64_CPUID_FN_7, 18);1000 msr_clear_bit(MSR_AMD64_CPUID_FN_7, 18);
1164 pr_emerg("RDSEED is not reliable on this platform; disabling.\n");1001 pr_emerg("RDSEED is not reliable on this platform; disabling.\n");
1165 }1002 }
1003+ 
1004+ /* Correct misconfigured CPUID on some clients. */
1005+ clear_cpu_cap(c, X86_FEATURE_INVLPGB);
1166}1006}
1167 1007 
1168static void init_amd_zen3(struct cpuinfo_x86 *c)1008static void init_amd_zen3(struct cpuinfo_x86 *c)
1169{1009{
1010+ if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) {
1011+ /*
1012+ * Zen3 (Fam19 model < 0x10) parts are not susceptible to
1013+ * Branch Type Confusion, but predate the allocation of the
1014+ * BTC_NO bit.
1015+ */
1016+ if (!cpu_has(c, X86_FEATURE_BTC_NO))
1017+ set_cpu_cap(c, X86_FEATURE_BTC_NO);
1018+ }
1170}1019}
1171 1020 
1172static void init_amd_zen4(struct cpuinfo_x86 *c)1021static void init_amd_zen4(struct cpuinfo_x86 *c)
1173{1022{
1023+ if (!cpu_has(c, X86_FEATURE_HYPERVISOR))
1024+ msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT);
1025+ 
1026+ /*
1027+ * These Zen4 SoCs advertise support for virtualized VMLOAD/VMSAVE
1028+ * in some BIOS versions but they can lead to random host reboots.
1029+ */
1030+ switch (c->x86_model) {
1031+ case 0x18 ... 0x1f:
1032+ case 0x60 ... 0x7f:
1033+ clear_cpu_cap(c, X86_FEATURE_V_VMSAVE_VMLOAD);
1034+ break;
1035+ }
1036+}
1037+ 
1038+static const struct x86_cpu_id zen5_rdseed_microcode[] = {
1039+ ZEN_MODEL_STEP_UCODE(0x1a, 0x02, 0x1, 0x0b00215a),
1040+ ZEN_MODEL_STEP_UCODE(0x1a, 0x11, 0x0, 0x0b101054),
1041+};
1042+ 
1043+static void init_amd_zen5(struct cpuinfo_x86 *c)
1044+{
1045+ if (!x86_match_min_microcode_rev(zen5_rdseed_microcode)) {
1046+ clear_cpu_cap(c, X86_FEATURE_RDSEED);
1047+ msr_clear_bit(MSR_AMD64_CPUID_FN_7, 18);
1048+ pr_emerg_once("RDSEED32 is broken. Disabling the corresponding CPUID bit.\n");
1049+ }
1174}1050}
1175 1051 
1176static void init_amd(struct cpuinfo_x86 *c)1052static void init_amd(struct cpuinfo_x86 *c)
1177{1053{
1054+ u64 vm_cr;
1055+ 
1178 early_init_amd(c);1056 early_init_amd(c);
1179 1057 
1180 /*1058 /*
@@ -1190,9 +1068,6 @@ static void init_amd(struct cpuinfo_x86 *c)
1190 if (cpu_has(c, X86_FEATURE_FSRM))1068 if (cpu_has(c, X86_FEATURE_FSRM))
1191 set_cpu_cap(c, X86_FEATURE_FSRS);1069 set_cpu_cap(c, X86_FEATURE_FSRS);
1192 1070 
1193- /* get apicid instead of initial apic id from cpuid */
1194- c->apicid = read_apic_id();
1195- 
1196 /* K6s reports MCEs but don't actually have all the MSRs */1071 /* K6s reports MCEs but don't actually have all the MSRs */
1197 if (c->x86 < 6)1072 if (c->x86 < 6)
1198 clear_cpu_cap(c, X86_FEATURE_MCE);1073 clear_cpu_cap(c, X86_FEATURE_MCE);
@@ -1206,11 +1081,15 @@ static void init_amd(struct cpuinfo_x86 *c)
1206 case 0x12: init_amd_ln(c); break;1081 case 0x12: init_amd_ln(c); break;
1207 case 0x15: init_amd_bd(c); break;1082 case 0x15: init_amd_bd(c); break;
1208 case 0x16: init_amd_jg(c); break;1083 case 0x16: init_amd_jg(c); break;
1209- case 0x17: init_spectral_chicken(c);
1210- fallthrough;
1211- case 0x19: init_amd_zn(c); break;
1212 }1084 }
1213 1085 
1086+ /*
1087+ * Save up on some future enablement work and do common Zen
1088+ * settings.
1089+ */
1090+ if (c->x86 >= 0x17)
1091+ init_amd_zen_common();
1092+ 
1214 if (boot_cpu_has(X86_FEATURE_ZEN1))1093 if (boot_cpu_has(X86_FEATURE_ZEN1))
1215 init_amd_zen1(c);1094 init_amd_zen1(c);
1216 else if (boot_cpu_has(X86_FEATURE_ZEN2))1095 else if (boot_cpu_has(X86_FEATURE_ZEN2))
@@ -1219,6 +1098,8 @@ static void init_amd(struct cpuinfo_x86 *c)
1219 init_amd_zen3(c);1098 init_amd_zen3(c);
1220 else if (boot_cpu_has(X86_FEATURE_ZEN4))1099 else if (boot_cpu_has(X86_FEATURE_ZEN4))
1221 init_amd_zen4(c);1100 init_amd_zen4(c);
1101+ else if (boot_cpu_has(X86_FEATURE_ZEN5))
1102+ init_amd_zen5(c);
1222 1103 
1223 /*1104 /*
1224 * Enable workaround for FXSAVE leak on CPUs1105 * Enable workaround for FXSAVE leak on CPUs
@@ -1229,12 +1110,18 @@ static void init_amd(struct cpuinfo_x86 *c)
1229 1110 
1230 cpu_detect_cache_sizes(c);1111 cpu_detect_cache_sizes(c);
1231 1112 
1232- amd_detect_cmp(c);
1233- amd_get_topology(c);
1234 srat_detect_node(c);1113 srat_detect_node(c);
1235 1114 
1236 init_amd_cacheinfo(c);1115 init_amd_cacheinfo(c);
1237 1116 
1117+ if (cpu_has(c, X86_FEATURE_SVM)) {
1118+ rdmsrq(MSR_VM_CR, vm_cr);
1119+ if (vm_cr & SVM_VM_CR_SVM_DIS_MASK) {
1120+ pr_notice_once("SVM disabled (by BIOS) in MSR_VM_CR\n");
1121+ clear_cpu_cap(c, X86_FEATURE_SVM);
1122+ }
1123+ }
1124+ 
1238 if (!cpu_has(c, X86_FEATURE_LFENCE_RDTSC) && cpu_has(c, X86_FEATURE_XMM2)) {1125 if (!cpu_has(c, X86_FEATURE_LFENCE_RDTSC) && cpu_has(c, X86_FEATURE_XMM2)) {
1239 /*1126 /*
1240 * Use LFENCE for execution serialization. On families which1127 * Use LFENCE for execution serialization. On families which
@@ -1282,12 +1169,12 @@ static void init_amd(struct cpuinfo_x86 *c)
1282 cpu_has(c, X86_FEATURE_AUTOIBRS))1169 cpu_has(c, X86_FEATURE_AUTOIBRS))
1283 WARN_ON_ONCE(msr_set_bit(MSR_EFER, _EFER_AUTOIBRS) < 0);1170 WARN_ON_ONCE(msr_set_bit(MSR_EFER, _EFER_AUTOIBRS) < 0);
1284 1171 
1285- if (!cpu_has(c, X86_FEATURE_HYPERVISOR) &&
1286- cpu_has_amd_erratum(c, amd_erratum_1485))
1287- msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT);
1288- 
1289 /* AMD CPUs don't need fencing after x2APIC/TSC_DEADLINE MSR writes. */1172 /* AMD CPUs don't need fencing after x2APIC/TSC_DEADLINE MSR writes. */
1290 clear_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);1173 clear_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
1174+ 
1175+ /* Enable Translation Cache Extension */
1176+ if (cpu_has(c, X86_FEATURE_TCE))
1177+ msr_set_bit(MSR_EFER, _EFER_TCE);
1291}1178}
1292 1179 
1293#ifdef CONFIG_X86_321180#ifdef CONFIG_X86_32
@@ -1320,8 +1207,8 @@ static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
1320 1207 
1321 cpuid(0x80000006, &eax, &ebx, &ecx, &edx);1208 cpuid(0x80000006, &eax, &ebx, &ecx, &edx);
1322 1209 
1323- tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask;1210+ tlb_lld_4k = (ebx >> 16) & mask;
1324- tlb_lli_4k[ENTRIES] = ebx & mask;1211+ tlb_lli_4k = ebx & mask;
1325 1212 
1326 /*1213 /*
1327 * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB1214 * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB
@@ -1334,26 +1221,30 @@ static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
1334 1221 
1335 /* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */1222 /* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
1336 if (!((eax >> 16) & mask))1223 if (!((eax >> 16) & mask))
1337- tlb_lld_2m[ENTRIES] = (cpuid_eax(0x80000005) >> 16) & 0xff;1224+ tlb_lld_2m = (cpuid_eax(0x80000005) >> 16) & 0xff;
1338 else1225 else
1339- tlb_lld_2m[ENTRIES] = (eax >> 16) & mask;1226+ tlb_lld_2m = (eax >> 16) & mask;
1340 1227 
1341 /* a 4M entry uses two 2M entries */1228 /* a 4M entry uses two 2M entries */
1342- tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1;1229+ tlb_lld_4m = tlb_lld_2m >> 1;
1343 1230 
1344 /* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */1231 /* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
1345 if (!(eax & mask)) {1232 if (!(eax & mask)) {
1346 /* Erratum 658 */1233 /* Erratum 658 */
1347 if (c->x86 == 0x15 && c->x86_model <= 0x1f) {1234 if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
1348- tlb_lli_2m[ENTRIES] = 1024;1235+ tlb_lli_2m = 1024;
1349 } else {1236 } else {
1350 cpuid(0x80000005, &eax, &ebx, &ecx, &edx);1237 cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
1351- tlb_lli_2m[ENTRIES] = eax & 0xff;1238+ tlb_lli_2m = eax & 0xff;
1352 }1239 }
1353 } else1240 } else
1354- tlb_lli_2m[ENTRIES] = eax & mask;1241+ tlb_lli_2m = eax & mask;
1355 1242 
1356- tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;1243+ tlb_lli_4m = tlb_lli_2m >> 1;
1244+ 
1245+ /* Max number of pages INVLPGB can invalidate in one shot */
1246+ if (cpu_has(c, X86_FEATURE_INVLPGB))
1247+ invlpgb_count_max = (cpuid_edx(0x80000008) & 0xffff) + 1;
1357}1248}
1358 1249 
1359static const struct cpu_dev amd_cpu_dev = {1250static const struct cpu_dev amd_cpu_dev = {
@@ -1405,7 +1296,7 @@ void amd_set_dr_addr_mask(unsigned long mask, unsigned int dr)
1405 if (per_cpu(amd_dr_addr_mask, cpu)[dr] == mask)1296 if (per_cpu(amd_dr_addr_mask, cpu)[dr] == mask)
1406 return;1297 return;
1407 1298 
1408- wrmsr(amd_msr_dr_addr_masks[dr], mask, 0);1299+ wrmsrq(amd_msr_dr_addr_masks[dr], mask);
1409 per_cpu(amd_dr_addr_mask, cpu)[dr] = mask;1300 per_cpu(amd_dr_addr_mask, cpu)[dr] = mask;
1410}1301}
1411 1302 
@@ -1421,22 +1312,6 @@ unsigned long amd_get_dr_addr_mask(unsigned int dr)
1421}1312}
1422EXPORT_SYMBOL_GPL(amd_get_dr_addr_mask);1313EXPORT_SYMBOL_GPL(amd_get_dr_addr_mask);
1423 1314 
1424-u32 amd_get_highest_perf(void)
1425-{
1426- struct cpuinfo_x86 *c = &boot_cpu_data;
1427- 
1428- if (c->x86 == 0x17 && ((c->x86_model >= 0x30 && c->x86_model < 0x40) ||
1429- (c->x86_model >= 0x70 && c->x86_model < 0x80)))
1430- return 166;
1431- 
1432- if (c->x86 == 0x19 && ((c->x86_model >= 0x20 && c->x86_model < 0x30) ||
1433- (c->x86_model >= 0x40 && c->x86_model < 0x70)))
1434- return 166;
1435- 
1436- return 255;
1437-}
1438-EXPORT_SYMBOL_GPL(amd_get_highest_perf);
1439- 
1440static void zenbleed_check_cpu(void *unused)1315static void zenbleed_check_cpu(void *unused)
1441{1316{
1442 struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());1317 struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());
@@ -1453,13 +1328,71 @@ void amd_check_microcode(void)
1453 on_each_cpu(zenbleed_check_cpu, NULL, 1);1328 on_each_cpu(zenbleed_check_cpu, NULL, 1);
1454}1329}
1455 1330 
1456-/*1331+static const char * const s5_reset_reason_txt[] = {
1457- * Issue a DIV 0/1 insn to clear any division data from previous DIV1332+ [0] = "thermal pin BP_THERMTRIP_L was tripped",
1458- * operations.1333+ [1] = "power button was pressed for 4 seconds",
1459- */1334+ [2] = "shutdown pin was tripped",
1460-void noinstr amd_clear_divider(void)1335+ [4] = "remote ASF power off command was received",
1336+ [9] = "internal CPU thermal limit was tripped",
1337+ [16] = "system reset pin BP_SYS_RST_L was tripped",
1338+ [17] = "software issued PCI reset",
1339+ [18] = "software wrote 0x4 to reset control register 0xCF9",
1340+ [19] = "software wrote 0x6 to reset control register 0xCF9",
1341+ [20] = "software wrote 0xE to reset control register 0xCF9",
1342+ [21] = "ACPI power state transition occurred",
1343+ [22] = "keyboard reset pin KB_RST_L was tripped",
1344+ [23] = "internal CPU shutdown event occurred",
1345+ [24] = "system failed to boot before failed boot timer expired",
1346+ [25] = "hardware watchdog timer expired",
1347+ [26] = "remote ASF reset command was received",
1348+ [27] = "an uncorrected error caused a data fabric sync flood event",
1349+ [29] = "FCH and MP1 failed warm reset handshake",
1350+ [30] = "a parity error occurred",
1351+ [31] = "a software sync flood event occurred",
1352+};
1353+ 
1354+static __init int print_s5_reset_status_mmio(void)
1461{1355{
1462- asm volatile(ALTERNATIVE("", "div %2\n\t", X86_BUG_DIV0)1356+ void __iomem *addr;
1463- :: "a" (0), "d" (0), "r" (1));1357+ u32 value;
1358+ int i;
1359+ 
1360+ if (!cpu_feature_enabled(X86_FEATURE_ZEN))
1361+ return 0;
1362+ 
1363+ addr = ioremap(FCH_PM_BASE + FCH_PM_S5_RESET_STATUS, sizeof(value));
1364+ if (!addr)
1365+ return 0;
1366+ 
1367+ value = ioread32(addr);
1368+ 
1369+ /* Value with "all bits set" is an error response and should be ignored. */
1370+ if (value == U32_MAX) {
1371+ iounmap(addr);
1372+ return 0;
1373+ }
1374+ 
1375+ /*
1376+ * Clear all reason bits so they won't be retained if the next reset
1377+ * does not update the register. Besides, some bits are never cleared by
1378+ * hardware so it's software's responsibility to clear them.
1379+ *
1380+ * Writing the value back effectively clears all reason bits as they are
1381+ * write-1-to-clear.
1382+ */
1383+ iowrite32(value, addr);
1384+ iounmap(addr);
1385+ 
1386+ for (i = 0; i < ARRAY_SIZE(s5_reset_reason_txt); i++) {
1387+ if (!(value & BIT(i)))
1388+ continue;
1389+ 
1390+ if (s5_reset_reason_txt[i]) {
1391+ pr_info("x86/amd: Previous system reset reason [0x%08x]: %s\n",
1392+ value, s5_reset_reason_txt[i]);
1393+ }
1394+ }
1395+ 
1396+ return 0;
1464}1397}
1465-EXPORT_SYMBOL_GPL(amd_clear_divider);1398+late_initcall(print_s5_reset_status_mmio);
March/x86/mm/pat/memtype.c+28-35
@@ -14,7 +14,7 @@
14 * memory ranges: uncached, write-combining, write-through, write-protected,14 * memory ranges: uncached, write-combining, write-through, write-protected,
15 * and the most commonly used and default attribute: write-back caching.15 * and the most commonly used and default attribute: write-back caching.
16 *16 *
17- * PAT support supercedes and augments MTRR support in a compatible fashion: MTRR is17+ * PAT support supersedes and augments MTRR support in a compatible fashion: MTRR is
18 * a hardware interface to enumerate a limited number of physical memory ranges18 * a hardware interface to enumerate a limited number of physical memory ranges
19 * and set their caching attributes explicitly, programmed into the CPU via MSRs.19 * and set their caching attributes explicitly, programmed into the CPU via MSRs.
20 * Even modern CPUs have MTRRs enabled - but these are typically not touched20 * Even modern CPUs have MTRRs enabled - but these are typically not touched
@@ -240,6 +240,8 @@ void pat_cpu_init(void)
240 }240 }
241 241 
242 wrmsrl(MSR_IA32_CR_PAT, pat_msr_val);242 wrmsrl(MSR_IA32_CR_PAT, pat_msr_val);
243+ 
244+ __flush_tlb_all();
243}245}
244 246 
245/**247/**
@@ -296,13 +298,8 @@ void __init pat_bp_init(void)
296 /*298 /*
297 * Xen PV doesn't allow to set PAT MSR, but all cache modes are299 * Xen PV doesn't allow to set PAT MSR, but all cache modes are
298 * supported.300 * supported.
299- * When running as TDX guest setting the PAT MSR won't work either
300- * due to the requirement to set CR0.CD when doing so. Rely on
301- * firmware to have set the PAT MSR correctly.
302 */301 */
303- if (pat_disabled ||302+ if (pat_disabled || cpu_feature_enabled(X86_FEATURE_XENPV)) {
304- cpu_feature_enabled(X86_FEATURE_XENPV) ||
305- cpu_feature_enabled(X86_FEATURE_TDX_GUEST)) {
306 init_cache_modes(pat_msr_val);303 init_cache_modes(pat_msr_val);
307 return;304 return;
308 }305 }
@@ -982,42 +979,29 @@ static int get_pat_info(struct vm_area_struct *vma, resource_size_t *paddr,
982 return -EINVAL;979 return -EINVAL;
983}980}
984 981 
985-int track_pfn_copy(struct vm_area_struct *dst_vma,982+/*
986- struct vm_area_struct *src_vma, unsigned long *pfn)983+ * track_pfn_copy is called when vma that is covering the pfnmap gets
984+ * copied through copy_page_range().
985+ *
986+ * If the vma has a linear pfn mapping for the entire range, we get the prot
987+ * from pte and reserve the entire vma range with single reserve_pfn_range call.
988+ */
989+int track_pfn_copy(struct vm_area_struct *vma)
987{990{
988- const unsigned long vma_size = src_vma->vm_end - src_vma->vm_start;
989 resource_size_t paddr;991 resource_size_t paddr;
992+ unsigned long vma_size = vma->vm_end - vma->vm_start;
990 pgprot_t pgprot;993 pgprot_t pgprot;
991- int rc;
992 994 
993- if (!(src_vma->vm_flags & VM_PAT))995+ if (vma->vm_flags & VM_PAT) {
994- return 0;996+ if (get_pat_info(vma, &paddr, &pgprot))
997+ return -EINVAL;
998+ /* reserve the whole chunk covered by vma. */
999+ return reserve_pfn_range(paddr, vma_size, &pgprot, 1);
1000+ }
995 1001 
996- /*
997- * Duplicate the PAT information for the dst VMA based on the src
998- * VMA.
999- */
1000- if (get_pat_info(src_vma, &paddr, &pgprot))
1001- return -EINVAL;
1002- rc = reserve_pfn_range(paddr, vma_size, &pgprot, 1);
1003- if (rc)
1004- return rc;
1005- 
1006- /* Reservation for the destination VMA succeeded. */
1007- vm_flags_set(dst_vma, VM_PAT);
1008- *pfn = PHYS_PFN(paddr);
1009 return 0;1002 return 0;
1010}1003}
1011 1004 
1012-void untrack_pfn_copy(struct vm_area_struct *dst_vma, unsigned long pfn)
1013-{
1014- untrack_pfn(dst_vma, pfn, dst_vma->vm_end - dst_vma->vm_start, true);
1015- /*
1016- * Reservation was freed, any copied page tables will get cleaned
1017- * up later, but without getting PAT involved again.
1018- */
1019-}
1020- 
1021/*1005/*
1022 * prot is passed in as a parameter for the new mapping. If the vma has1006 * prot is passed in as a parameter for the new mapping. If the vma has
1023 * a linear pfn mapping for the entire range, or no vma is provided,1007 * a linear pfn mapping for the entire range, or no vma is provided,
@@ -1106,6 +1090,15 @@ void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
1106 }1090 }
1107}1091}
1108 1092 
1093+/*
1094+ * untrack_pfn_clear is called if the following situation fits:
1095+ *
1096+ * 1) while mremapping a pfnmap for a new region, with the old vma after
1097+ * its pfnmap page table has been removed. The new vma has a new pfnmap
1098+ * to the same pfn & cache type with VM_PAT set.
1099+ * 2) while duplicating vm area, the new vma fails to copy the pgtable from
1100+ * old vma.
1101+ */
1109void untrack_pfn_clear(struct vm_area_struct *vma)1102void untrack_pfn_clear(struct vm_area_struct *vma)
1110{1103{
1111 vm_flags_clear(vma, VM_PAT);1104 vm_flags_clear(vma, VM_PAT);
Mdrivers/accessibility/speakup/main.c+1-1
@@ -574,7 +574,7 @@ static u_long get_word(struct vc_data *vc)
574 }574 }
575 attr_ch = get_char(vc, (u_short *)tmp_pos, &spk_attr);575 attr_ch = get_char(vc, (u_short *)tmp_pos, &spk_attr);
576 buf[cnt++] = attr_ch;576 buf[cnt++] = attr_ch;
577- while (tmpx < vc->vc_cols - 1 && cnt < ARRAY_SIZE(buf) - 1) {577+ while (tmpx < vc->vc_cols - 1 && cnt < sizeof(buf) - 1) {
578 tmp_pos += 2;578 tmp_pos += 2;
579 tmpx++;579 tmpx++;
580 ch = get_char(vc, (u_short *)tmp_pos, &temp);580 ch = get_char(vc, (u_short *)tmp_pos, &temp);
Mdrivers/crypto/aspeed/aspeed-acry.c+1-7
@@ -601,8 +601,6 @@ static struct aspeed_acry_alg aspeed_acry_akcipher_algs[] = {
601 .akcipher.base = {601 .akcipher.base = {
602 .encrypt = aspeed_acry_rsa_enc,602 .encrypt = aspeed_acry_rsa_enc,
603 .decrypt = aspeed_acry_rsa_dec,603 .decrypt = aspeed_acry_rsa_dec,
604- .sign = aspeed_acry_rsa_dec,
605- .verify = aspeed_acry_rsa_enc,
606 .set_pub_key = aspeed_acry_rsa_set_pub_key,604 .set_pub_key = aspeed_acry_rsa_set_pub_key,
607 .set_priv_key = aspeed_acry_rsa_set_priv_key,605 .set_priv_key = aspeed_acry_rsa_set_priv_key,
608 .max_size = aspeed_acry_rsa_max_size,606 .max_size = aspeed_acry_rsa_max_size,
@@ -789,21 +787,17 @@ static int aspeed_acry_probe(struct platform_device *pdev)
789err_engine_rsa_start:787err_engine_rsa_start:
790 crypto_engine_exit(acry_dev->crypt_engine_rsa);788 crypto_engine_exit(acry_dev->crypt_engine_rsa);
791clk_exit:789clk_exit:
792- clk_disable_unprepare(acry_dev->clk);
793 790 
794 return rc;791 return rc;
795}792}
796 793 
797-static int aspeed_acry_remove(struct platform_device *pdev)794+static void aspeed_acry_remove(struct platform_device *pdev)
798{795{
799 struct aspeed_acry_dev *acry_dev = platform_get_drvdata(pdev);796 struct aspeed_acry_dev *acry_dev = platform_get_drvdata(pdev);
800 797 
801 aspeed_acry_unregister(acry_dev);798 aspeed_acry_unregister(acry_dev);
802 crypto_engine_exit(acry_dev->crypt_engine_rsa);799 crypto_engine_exit(acry_dev->crypt_engine_rsa);
803 tasklet_kill(&acry_dev->done_task);800 tasklet_kill(&acry_dev->done_task);
804- clk_disable_unprepare(acry_dev->clk);
805- 
806- return 0;
807}801}
808 802 
809MODULE_DEVICE_TABLE(of, aspeed_acry_of_matches);803MODULE_DEVICE_TABLE(of, aspeed_acry_of_matches);
Mdrivers/gpu/drm/amd/amdkfd/kfd_device.c+407-85
@@ -56,6 +56,7 @@ extern const struct kfd2kgd_calls gc_9_4_3_kfd2kgd;
56extern const struct kfd2kgd_calls gfx_v10_kfd2kgd;56extern const struct kfd2kgd_calls gfx_v10_kfd2kgd;
57extern const struct kfd2kgd_calls gfx_v10_3_kfd2kgd;57extern const struct kfd2kgd_calls gfx_v10_3_kfd2kgd;
58extern const struct kfd2kgd_calls gfx_v11_kfd2kgd;58extern const struct kfd2kgd_calls gfx_v11_kfd2kgd;
59+extern const struct kfd2kgd_calls gfx_v12_kfd2kgd;
59 60 
60static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,61static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
61 unsigned int chunk_size);62 unsigned int chunk_size);
@@ -65,7 +66,7 @@ static int kfd_resume(struct kfd_node *kfd);
65 66 
66static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd)67static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd)
67{68{
68- uint32_t sdma_version = kfd->adev->ip_versions[SDMA0_HWIP][0];69+ uint32_t sdma_version = amdgpu_ip_version(kfd->adev, SDMA0_HWIP, 0);
69 70 
70 switch (sdma_version) {71 switch (sdma_version) {
71 case IP_VERSION(4, 0, 0):/* VEGA10 */72 case IP_VERSION(4, 0, 0):/* VEGA10 */
@@ -83,6 +84,8 @@ static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd)
83 case IP_VERSION(4, 2, 2):/* ARCTURUS */84 case IP_VERSION(4, 2, 2):/* ARCTURUS */
84 case IP_VERSION(4, 4, 0):/* ALDEBARAN */85 case IP_VERSION(4, 4, 0):/* ALDEBARAN */
85 case IP_VERSION(4, 4, 2):86 case IP_VERSION(4, 4, 2):
87+ case IP_VERSION(4, 4, 5):
88+ case IP_VERSION(4, 4, 4):
86 case IP_VERSION(5, 0, 0):/* NAVI10 */89 case IP_VERSION(5, 0, 0):/* NAVI10 */
87 case IP_VERSION(5, 0, 1):/* CYAN_SKILLFISH */90 case IP_VERSION(5, 0, 1):/* CYAN_SKILLFISH */
88 case IP_VERSION(5, 0, 2):/* NAVI14 */91 case IP_VERSION(5, 0, 2):/* NAVI14 */
@@ -95,6 +98,12 @@ static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd)
95 case IP_VERSION(6, 0, 1):98 case IP_VERSION(6, 0, 1):
96 case IP_VERSION(6, 0, 2):99 case IP_VERSION(6, 0, 2):
97 case IP_VERSION(6, 0, 3):100 case IP_VERSION(6, 0, 3):
101+ case IP_VERSION(6, 1, 0):
102+ case IP_VERSION(6, 1, 1):
103+ case IP_VERSION(6, 1, 2):
104+ case IP_VERSION(6, 1, 3):
105+ case IP_VERSION(7, 0, 0):
106+ case IP_VERSION(7, 0, 1):
98 kfd->device_info.num_sdma_queues_per_engine = 8;107 kfd->device_info.num_sdma_queues_per_engine = 8;
99 break;108 break;
100 default:109 default:
@@ -111,6 +120,12 @@ static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd)
111 case IP_VERSION(6, 0, 1):120 case IP_VERSION(6, 0, 1):
112 case IP_VERSION(6, 0, 2):121 case IP_VERSION(6, 0, 2):
113 case IP_VERSION(6, 0, 3):122 case IP_VERSION(6, 0, 3):
123+ case IP_VERSION(6, 1, 0):
124+ case IP_VERSION(6, 1, 1):
125+ case IP_VERSION(6, 1, 2):
126+ case IP_VERSION(6, 1, 3):
127+ case IP_VERSION(7, 0, 0):
128+ case IP_VERSION(7, 0, 1):
114 /* Reserve 1 for paging and 1 for gfx */129 /* Reserve 1 for paging and 1 for gfx */
115 kfd->device_info.num_reserved_sdma_queues_per_engine = 2;130 kfd->device_info.num_reserved_sdma_queues_per_engine = 2;
116 /* BIT(0)=engine-0 queue-0; BIT(1)=engine-1 queue-0; BIT(2)=engine-0 queue-1; ... */131 /* BIT(0)=engine-0 queue-0; BIT(1)=engine-1 queue-0; BIT(2)=engine-0 queue-1; ... */
@@ -139,6 +154,8 @@ static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd)
139 kfd->device_info.event_interrupt_class = &event_interrupt_class_v9;154 kfd->device_info.event_interrupt_class = &event_interrupt_class_v9;
140 break;155 break;
141 case IP_VERSION(9, 4, 3): /* GC 9.4.3 */156 case IP_VERSION(9, 4, 3): /* GC 9.4.3 */
157+ case IP_VERSION(9, 4, 4): /* GC 9.4.4 */
158+ case IP_VERSION(9, 5, 0): /* GC 9.5.0 */
142 kfd->device_info.event_interrupt_class =159 kfd->device_info.event_interrupt_class =
143 &event_interrupt_class_v9_4_3;160 &event_interrupt_class_v9_4_3;
144 break;161 break;
@@ -162,6 +179,15 @@ static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd)
162 case IP_VERSION(11, 0, 2):179 case IP_VERSION(11, 0, 2):
163 case IP_VERSION(11, 0, 3):180 case IP_VERSION(11, 0, 3):
164 case IP_VERSION(11, 0, 4):181 case IP_VERSION(11, 0, 4):
182+ case IP_VERSION(11, 5, 0):
183+ case IP_VERSION(11, 5, 1):
184+ case IP_VERSION(11, 5, 2):
185+ case IP_VERSION(11, 5, 3):
186+ kfd->device_info.event_interrupt_class = &event_interrupt_class_v11;
187+ break;
188+ case IP_VERSION(12, 0, 0):
189+ case IP_VERSION(12, 0, 1):
190+ /* GFX12_TODO: Change to v12 version. */
165 kfd->device_info.event_interrupt_class = &event_interrupt_class_v11;191 kfd->device_info.event_interrupt_class = &event_interrupt_class_v11;
166 break;192 break;
167 default:193 default:
@@ -214,6 +240,11 @@ static void kfd_device_info_init(struct kfd_dev *kfd,
214 */240 */
215 kfd->device_info.needs_pci_atomics = true;241 kfd->device_info.needs_pci_atomics = true;
216 kfd->device_info.no_atomic_fw_version = kfd->adev->gfx.rs64_enable ? 509 : 0;242 kfd->device_info.no_atomic_fw_version = kfd->adev->gfx.rs64_enable ? 509 : 0;
243+ } else if (gc_version < IP_VERSION(13, 0, 0)) {
244+ kfd->device_info.needs_pci_atomics = true;
245+ kfd->device_info.no_atomic_fw_version = 2090;
246+ } else {
247+ kfd->device_info.needs_pci_atomics = true;
217 }248 }
218 } else {249 } else {
219 kfd->device_info.doorbell_size = 4;250 kfd->device_info.doorbell_size = 4;
@@ -279,7 +310,7 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)
279 f2g = &gfx_v8_kfd2kgd;310 f2g = &gfx_v8_kfd2kgd;
280 break;311 break;
281 default:312 default:
282- switch (adev->ip_versions[GC_HWIP][0]) {313+ switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
283 /* Vega 10 */314 /* Vega 10 */
284 case IP_VERSION(9, 0, 1):315 case IP_VERSION(9, 0, 1):
285 gfx_target_version = 90000;316 gfx_target_version = 90000;
@@ -321,9 +352,12 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)
321 f2g = &aldebaran_kfd2kgd;352 f2g = &aldebaran_kfd2kgd;
322 break;353 break;
323 case IP_VERSION(9, 4, 3):354 case IP_VERSION(9, 4, 3):
324- gfx_target_version = adev->rev_id >= 1 ? 90402355+ case IP_VERSION(9, 4, 4):
325- : adev->flags & AMD_IS_APU ? 90400356+ gfx_target_version = 90402;
326- : 90401;357+ f2g = &gc_9_4_3_kfd2kgd;
358+ break;
359+ case IP_VERSION(9, 5, 0):
360+ gfx_target_version = 90500;
327 f2g = &gc_9_4_3_kfd2kgd;361 f2g = &gc_9_4_3_kfd2kgd;
328 break;362 break;
329 /* Navi10 */363 /* Navi10 */
@@ -406,6 +440,30 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)
406 gfx_target_version = 110001;440 gfx_target_version = 110001;
407 f2g = &gfx_v11_kfd2kgd;441 f2g = &gfx_v11_kfd2kgd;
408 break;442 break;
443+ case IP_VERSION(11, 5, 0):
444+ gfx_target_version = 110500;
445+ f2g = &gfx_v11_kfd2kgd;
446+ break;
447+ case IP_VERSION(11, 5, 1):
448+ gfx_target_version = 110501;
449+ f2g = &gfx_v11_kfd2kgd;
450+ break;
451+ case IP_VERSION(11, 5, 2):
452+ gfx_target_version = 110502;
453+ f2g = &gfx_v11_kfd2kgd;
454+ break;
455+ case IP_VERSION(11, 5, 3):
456+ gfx_target_version = 110503;
457+ f2g = &gfx_v11_kfd2kgd;
458+ break;
459+ case IP_VERSION(12, 0, 0):
460+ gfx_target_version = 120000;
461+ f2g = &gfx_v12_kfd2kgd;
462+ break;
463+ case IP_VERSION(12, 0, 1):
464+ gfx_target_version = 120001;
465+ f2g = &gfx_v12_kfd2kgd;
466+ break;
409 default:467 default:
410 break;468 break;
411 }469 }
@@ -413,11 +471,13 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)
413 }471 }
414 472 
415 if (!f2g) {473 if (!f2g) {
416- if (adev->ip_versions[GC_HWIP][0])474+ if (amdgpu_ip_version(adev, GC_HWIP, 0))
417- dev_err(kfd_device, "GC IP %06x %s not supported in kfd\n",475+ dev_info(kfd_device,
418- adev->ip_versions[GC_HWIP][0], vf ? "VF" : "");476+ "GC IP %06x %s not supported in kfd\n",
477+ amdgpu_ip_version(adev, GC_HWIP, 0),
478+ vf ? "VF" : "");
419 else479 else
420- dev_err(kfd_device, "%s %s not supported in kfd\n",480+ dev_info(kfd_device, "%s %s not supported in kfd\n",
421 amdgpu_asic_name[adev->asic_type], vf ? "VF" : "");481 amdgpu_asic_name[adev->asic_type], vf ? "VF" : "");
422 return NULL;482 return NULL;
423 }483 }
@@ -435,6 +495,7 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)
435 mutex_init(&kfd->doorbell_mutex);495 mutex_init(&kfd->doorbell_mutex);
436 496 
437 ida_init(&kfd->doorbell_ida);497 ida_init(&kfd->doorbell_ida);
498+ atomic_set(&kfd->kfd_processes_count, 0);
438 499 
439 return kfd;500 return kfd;
440}501}
@@ -443,37 +504,56 @@ static void kfd_cwsr_init(struct kfd_dev *kfd)
443{504{
444 if (cwsr_enable && kfd->device_info.supports_cwsr) {505 if (cwsr_enable && kfd->device_info.supports_cwsr) {
445 if (KFD_GC_VERSION(kfd) < IP_VERSION(9, 0, 1)) {506 if (KFD_GC_VERSION(kfd) < IP_VERSION(9, 0, 1)) {
446- BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > PAGE_SIZE);507+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex)
508+ > KFD_CWSR_TMA_OFFSET);
447 kfd->cwsr_isa = cwsr_trap_gfx8_hex;509 kfd->cwsr_isa = cwsr_trap_gfx8_hex;
448 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex);510 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex);
449 } else if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 1)) {511 } else if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 1)) {
450- BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) > PAGE_SIZE);512+ BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex)
513+ > KFD_CWSR_TMA_OFFSET);
451 kfd->cwsr_isa = cwsr_trap_arcturus_hex;514 kfd->cwsr_isa = cwsr_trap_arcturus_hex;
452 kfd->cwsr_isa_size = sizeof(cwsr_trap_arcturus_hex);515 kfd->cwsr_isa_size = sizeof(cwsr_trap_arcturus_hex);
453 } else if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 2)) {516 } else if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 2)) {
454- BUILD_BUG_ON(sizeof(cwsr_trap_aldebaran_hex) > PAGE_SIZE);517+ BUILD_BUG_ON(sizeof(cwsr_trap_aldebaran_hex)
518+ > KFD_CWSR_TMA_OFFSET);
455 kfd->cwsr_isa = cwsr_trap_aldebaran_hex;519 kfd->cwsr_isa = cwsr_trap_aldebaran_hex;
456 kfd->cwsr_isa_size = sizeof(cwsr_trap_aldebaran_hex);520 kfd->cwsr_isa_size = sizeof(cwsr_trap_aldebaran_hex);
457- } else if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 3)) {521+ } else if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 3) ||
458- BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_4_3_hex) > PAGE_SIZE);522+ KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 4)) {
523+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_4_3_hex)
524+ > KFD_CWSR_TMA_OFFSET);
459 kfd->cwsr_isa = cwsr_trap_gfx9_4_3_hex;525 kfd->cwsr_isa = cwsr_trap_gfx9_4_3_hex;
460 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_4_3_hex);526 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_4_3_hex);
527+ } else if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 5, 0)) {
528+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_5_0_hex) > PAGE_SIZE);
529+ kfd->cwsr_isa = cwsr_trap_gfx9_5_0_hex;
530+ kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_5_0_hex);
461 } else if (KFD_GC_VERSION(kfd) < IP_VERSION(10, 1, 1)) {531 } else if (KFD_GC_VERSION(kfd) < IP_VERSION(10, 1, 1)) {
462- BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) > PAGE_SIZE);532+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex)
533+ > KFD_CWSR_TMA_OFFSET);
463 kfd->cwsr_isa = cwsr_trap_gfx9_hex;534 kfd->cwsr_isa = cwsr_trap_gfx9_hex;
464 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex);535 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex);
465 } else if (KFD_GC_VERSION(kfd) < IP_VERSION(10, 3, 0)) {536 } else if (KFD_GC_VERSION(kfd) < IP_VERSION(10, 3, 0)) {
466- BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex) > PAGE_SIZE);537+ BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex)
538+ > KFD_CWSR_TMA_OFFSET);
467 kfd->cwsr_isa = cwsr_trap_nv1x_hex;539 kfd->cwsr_isa = cwsr_trap_nv1x_hex;
468 kfd->cwsr_isa_size = sizeof(cwsr_trap_nv1x_hex);540 kfd->cwsr_isa_size = sizeof(cwsr_trap_nv1x_hex);
469 } else if (KFD_GC_VERSION(kfd) < IP_VERSION(11, 0, 0)) {541 } else if (KFD_GC_VERSION(kfd) < IP_VERSION(11, 0, 0)) {
470- BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) > PAGE_SIZE);542+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex)
543+ > KFD_CWSR_TMA_OFFSET);
471 kfd->cwsr_isa = cwsr_trap_gfx10_hex;544 kfd->cwsr_isa = cwsr_trap_gfx10_hex;
472 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx10_hex);545 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx10_hex);
473- } else {546+ } else if (KFD_GC_VERSION(kfd) < IP_VERSION(12, 0, 0)) {
547+ /* The gfx11 cwsr trap handler must fit inside a single
548+ page. */
474 BUILD_BUG_ON(sizeof(cwsr_trap_gfx11_hex) > PAGE_SIZE);549 BUILD_BUG_ON(sizeof(cwsr_trap_gfx11_hex) > PAGE_SIZE);
475 kfd->cwsr_isa = cwsr_trap_gfx11_hex;550 kfd->cwsr_isa = cwsr_trap_gfx11_hex;
476 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx11_hex);551 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx11_hex);
552+ } else {
553+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_hex)
554+ > KFD_CWSR_TMA_OFFSET);
555+ kfd->cwsr_isa = cwsr_trap_gfx12_hex;
556+ kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx12_hex);
477 }557 }
478 558 
479 kfd->cwsr_enabled = true;559 kfd->cwsr_enabled = true;
@@ -498,15 +578,21 @@ static int kfd_gws_init(struct kfd_node *node)
498 && kfd->mec2_fw_version >= 0x30) ||578 && kfd->mec2_fw_version >= 0x30) ||
499 (KFD_GC_VERSION(node) == IP_VERSION(9, 4, 2)579 (KFD_GC_VERSION(node) == IP_VERSION(9, 4, 2)
500 && kfd->mec2_fw_version >= 0x28) ||580 && kfd->mec2_fw_version >= 0x28) ||
501- (KFD_GC_VERSION(node) == IP_VERSION(9, 4, 3)) ||581+ (KFD_GC_VERSION(node) == IP_VERSION(9, 4, 3) ||
582+ KFD_GC_VERSION(node) == IP_VERSION(9, 4, 4)) ||
583+ (KFD_GC_VERSION(node) == IP_VERSION(9, 5, 0)) ||
502 (KFD_GC_VERSION(node) >= IP_VERSION(10, 3, 0)584 (KFD_GC_VERSION(node) >= IP_VERSION(10, 3, 0)
503 && KFD_GC_VERSION(node) < IP_VERSION(11, 0, 0)585 && KFD_GC_VERSION(node) < IP_VERSION(11, 0, 0)
504 && kfd->mec2_fw_version >= 0x6b) ||586 && kfd->mec2_fw_version >= 0x6b) ||
505 (KFD_GC_VERSION(node) >= IP_VERSION(11, 0, 0)587 (KFD_GC_VERSION(node) >= IP_VERSION(11, 0, 0)
506 && KFD_GC_VERSION(node) < IP_VERSION(12, 0, 0)588 && KFD_GC_VERSION(node) < IP_VERSION(12, 0, 0)
507- && mes_rev >= 68))))589+ && mes_rev >= 68) ||
590+ (KFD_GC_VERSION(node) >= IP_VERSION(12, 0, 0))))) {
591+ if (KFD_GC_VERSION(node) >= IP_VERSION(12, 0, 0))
592+ node->adev->gds.gws_size = 64;
508 ret = amdgpu_amdkfd_alloc_gws(node->adev,593 ret = amdgpu_amdkfd_alloc_gws(node->adev,
509 node->adev->gds.gws_size, &node->gws);594 node->adev->gds.gws_size, &node->gws);
595+ }
510 596 
511 return ret;597 return ret;
512}598}
@@ -570,6 +656,14 @@ static void kfd_cleanup_nodes(struct kfd_dev *kfd, unsigned int num_nodes)
570 struct kfd_node *knode;656 struct kfd_node *knode;
571 unsigned int i;657 unsigned int i;
572 658 
659+ /*
660+ * flush_work ensures that there are no outstanding
661+ * work-queue items that will access interrupt_ring. New work items
662+ * can't be created because we stopped interrupt handling above.
663+ */
664+ flush_workqueue(kfd->ih_wq);
665+ destroy_workqueue(kfd->ih_wq);
666+ 
573 for (i = 0; i < num_nodes; i++) {667 for (i = 0; i < num_nodes; i++) {
574 knode = kfd->nodes[i];668 knode = kfd->nodes[i];
575 device_queue_manager_uninit(knode->dqm);669 device_queue_manager_uninit(knode->dqm);
@@ -665,14 +759,14 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
665 last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1;759 last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1;
666 vmid_num_kfd = last_vmid_kfd - first_vmid_kfd + 1;760 vmid_num_kfd = last_vmid_kfd - first_vmid_kfd + 1;
667 761 
668- /* For GFX9.4.3, we need special handling for VMIDs depending on762+ /* For multi-partition capable GPUs, we need special handling for VMIDs
669- * partition mode.763+ * depending on partition mode.
670 * In CPX mode, the VMID range needs to be shared between XCDs.764 * In CPX mode, the VMID range needs to be shared between XCDs.
671 * Additionally, there are 13 VMIDs (3-15) available for KFD. To765 * Additionally, there are 13 VMIDs (3-15) available for KFD. To
672 * divide them equally, we change starting VMID to 4 and not use766 * divide them equally, we change starting VMID to 4 and not use
673 * VMID 3.767 * VMID 3.
674- * If the VMID range changes for GFX9.4.3, then this code MUST be768+ * If the VMID range changes for multi-partition capable GPUs, then
675- * revisited.769+ * this code MUST be revisited.
676 */770 */
677 if (kfd->adev->xcp_mgr) {771 if (kfd->adev->xcp_mgr) {
678 partition_mode = amdgpu_xcp_query_partition_mode(kfd->adev->xcp_mgr,772 partition_mode = amdgpu_xcp_query_partition_mode(kfd->adev->xcp_mgr,
@@ -737,11 +831,12 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
737 kfd->hive_id = kfd->adev->gmc.xgmi.hive_id;831 kfd->hive_id = kfd->adev->gmc.xgmi.hive_id;
738 832 
739 /*833 /*
740- * For GFX9.4.3, the KFD abstracts all partitions within a socket as834+ * For multi-partition capable GPUs, the KFD abstracts all partitions
741- * xGMI connected in the topology so assign a unique hive id per835+ * within a socket as xGMI connected in the topology so assign a unique
742- * device based on the pci device location if device is in PCIe mode.836+ * hive id per device based on the pci device location if device is in
837+ * PCIe mode.
743 */838 */
744- if (!kfd->hive_id && (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 3)) && kfd->num_nodes > 1)839+ if (!kfd->hive_id && kfd->num_nodes > 1)
745 kfd->hive_id = pci_dev_id(kfd->adev->pdev);840 kfd->hive_id = pci_dev_id(kfd->adev->pdev);
746 841 
747 kfd->noretry = kfd->adev->gmc.noretry;842 kfd->noretry = kfd->adev->gmc.noretry;
@@ -779,11 +874,11 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
779 KFD_XCP_MEMORY_SIZE(node->adev, node->node_id) >> 20);874 KFD_XCP_MEMORY_SIZE(node->adev, node->node_id) >> 20);
780 }875 }
781 876 
782- if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 3) &&877+ if (partition_mode == AMDGPU_CPX_PARTITION_MODE &&
783- partition_mode == AMDGPU_CPX_PARTITION_MODE &&
784 kfd->num_nodes != 1) {878 kfd->num_nodes != 1) {
785- /* For GFX9.4.3 and CPX mode, first XCD gets VMID range879+ /* For multi-partition capable GPUs and CPX mode, first
786- * 4-9 and second XCD gets VMID range 10-15.880+ * XCD gets VMID range 4-9 and second XCD gets VMID
881+ * range 10-15.
787 */882 */
788 883 
789 node->vm_info.first_vmid_kfd = (i%2 == 0) ?884 node->vm_info.first_vmid_kfd = (i%2 == 0) ?
@@ -807,7 +902,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
807 amdgpu_amdkfd_get_local_mem_info(kfd->adev,902 amdgpu_amdkfd_get_local_mem_info(kfd->adev,
808 &node->local_mem_info, node->xcp);903 &node->local_mem_info, node->xcp);
809 904 
810- if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 3))905+ if (kfd->adev->xcp_mgr)
811 kfd_setup_interrupt_bitmap(node, i);906 kfd_setup_interrupt_bitmap(node, i);
812 907 
813 /* Initialize the KFD node */908 /* Initialize the KFD node */
@@ -815,13 +910,14 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
815 dev_err(kfd_device, "Error initializing KFD node\n");910 dev_err(kfd_device, "Error initializing KFD node\n");
816 goto node_init_error;911 goto node_init_error;
817 }912 }
913+ 
914+ spin_lock_init(&node->watch_points_lock);
915+ 
818 kfd->nodes[i] = node;916 kfd->nodes[i] = node;
819 }917 }
820 918 
821 svm_range_set_max_pages(kfd->adev);919 svm_range_set_max_pages(kfd->adev);
822 920 
823- spin_lock_init(&kfd->watch_points_lock);
824- 
825 kfd->init_complete = true;921 kfd->init_complete = true;
826 dev_info(kfd_device, "added device %x:%x\n", kfd->adev->pdev->vendor,922 dev_info(kfd_device, "added device %x:%x\n", kfd->adev->pdev->vendor,
827 kfd->adev->pdev->device);923 kfd->adev->pdev->device);
@@ -862,7 +958,8 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
862 kfree(kfd);958 kfree(kfd);
863}959}
864 960 
865-int kgd2kfd_pre_reset(struct kfd_dev *kfd)961+int kgd2kfd_pre_reset(struct kfd_dev *kfd,
962+ struct amdgpu_reset_context *reset_context)
866{963{
867 struct kfd_node *node;964 struct kfd_node *node;
868 int i;965 int i;
@@ -872,11 +969,10 @@ int kgd2kfd_pre_reset(struct kfd_dev *kfd)
872 969 
873 for (i = 0; i < kfd->num_nodes; i++) {970 for (i = 0; i < kfd->num_nodes; i++) {
874 node = kfd->nodes[i];971 node = kfd->nodes[i];
875- kfd_smi_event_update_gpu_reset(node, false);972+ kfd_smi_event_update_gpu_reset(node, false, reset_context);
876- node->dqm->ops.pre_reset(node->dqm);
877 }973 }
878 974 
879- kgd2kfd_suspend(kfd, false);975+ kgd2kfd_suspend(kfd, true);
880 976 
881 for (i = 0; i < kfd->num_nodes; i++)977 for (i = 0; i < kfd->num_nodes; i++)
882 kfd_signal_reset_event(kfd->nodes[i]);978 kfd_signal_reset_event(kfd->nodes[i]);
@@ -912,19 +1008,39 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
912 for (i = 0; i < kfd->num_nodes; i++) {1008 for (i = 0; i < kfd->num_nodes; i++) {
913 node = kfd->nodes[i];1009 node = kfd->nodes[i];
914 atomic_set(&node->sram_ecc_flag, 0);1010 atomic_set(&node->sram_ecc_flag, 0);
915- kfd_smi_event_update_gpu_reset(node, true);1011+ kfd_smi_event_update_gpu_reset(node, true, NULL);
916 }1012 }
917 1013 
918 return 0;1014 return 0;
919}1015}
920 1016 
921-bool kfd_is_locked(void)1017+bool kfd_is_locked(struct kfd_dev *kfd)
922{1018{
1019+ uint8_t id = 0;
1020+ struct kfd_node *dev;
1021+ 
923 lockdep_assert_held(&kfd_processes_mutex);1022 lockdep_assert_held(&kfd_processes_mutex);
924- return (kfd_locked > 0);1023+ 
1024+ /* check reset/suspend lock */
1025+ if (kfd_locked > 0)
1026+ return true;
1027+ 
1028+ if (kfd)
1029+ return kfd->kfd_dev_lock > 0;
1030+ 
1031+ /* check lock on all cgroup accessible devices */
1032+ while (kfd_topology_enum_kfd_devices(id++, &dev) == 0) {
1033+ if (!dev || kfd_devcgroup_check_permission(dev))
1034+ continue;
1035+ 
1036+ if (dev->kfd->kfd_dev_lock > 0)
1037+ return true;
1038+ }
1039+ 
1040+ return false;
925}1041}
926 1042 
927-void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm)1043+void kgd2kfd_suspend(struct kfd_dev *kfd, bool suspend_proc)
928{1044{
929 struct kfd_node *node;1045 struct kfd_node *node;
930 int i;1046 int i;
@@ -932,14 +1048,8 @@ void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm)
932 if (!kfd->init_complete)1048 if (!kfd->init_complete)
933 return;1049 return;
934 1050 
935- /* for runtime suspend, skip locking kfd */1051+ if (suspend_proc)
936- if (!run_pm) {1052+ kgd2kfd_suspend_process(kfd);
937- mutex_lock(&kfd_processes_mutex);
938- /* For first KFD device suspend all the KFD processes */
939- if (++kfd_locked == 1)
940- kfd_suspend_all_processes();
941- mutex_unlock(&kfd_processes_mutex);
942- }
943 1053 
944 for (i = 0; i < kfd->num_nodes; i++) {1054 for (i = 0; i < kfd->num_nodes; i++) {
945 node = kfd->nodes[i];1055 node = kfd->nodes[i];
@@ -947,7 +1057,7 @@ void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm)
947 }1057 }
948}1058}
949 1059 
950-int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm)1060+int kgd2kfd_resume(struct kfd_dev *kfd, bool resume_proc)
951{1061{
952 int ret, i;1062 int ret, i;
953 1063 
@@ -960,14 +1070,36 @@ int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm)
960 return ret;1070 return ret;
961 }1071 }
962 1072 
963- /* for runtime resume, skip unlocking kfd */1073+ if (resume_proc)
964- if (!run_pm) {1074+ ret = kgd2kfd_resume_process(kfd);
965- mutex_lock(&kfd_processes_mutex);1075+ 
966- if (--kfd_locked == 0)1076+ return ret;
967- ret = kfd_resume_all_processes();1077+}
968- WARN_ONCE(kfd_locked < 0, "KFD suspend / resume ref. error");1078+ 
969- mutex_unlock(&kfd_processes_mutex);1079+void kgd2kfd_suspend_process(struct kfd_dev *kfd)
970- }1080+{
1081+ if (!kfd->init_complete)
1082+ return;
1083+ 
1084+ mutex_lock(&kfd_processes_mutex);
1085+ /* For first KFD device suspend all the KFD processes */
1086+ if (++kfd_locked == 1)
1087+ kfd_suspend_all_processes();
1088+ mutex_unlock(&kfd_processes_mutex);
1089+}
1090+ 
1091+int kgd2kfd_resume_process(struct kfd_dev *kfd)
1092+{
1093+ int ret = 0;
1094+ 
1095+ if (!kfd->init_complete)
1096+ return 0;
1097+ 
1098+ mutex_lock(&kfd_processes_mutex);
1099+ if (--kfd_locked == 0)
1100+ ret = kfd_resume_all_processes();
1101+ WARN_ONCE(kfd_locked < 0, "KFD suspend / resume ref. error");
1102+ mutex_unlock(&kfd_processes_mutex);
971 1103 
972 return ret;1104 return ret;
973}1105}
@@ -985,21 +1117,6 @@ static int kfd_resume(struct kfd_node *node)
985 return err;1117 return err;
986}1118}
987 1119 
988-static inline void kfd_queue_work(struct workqueue_struct *wq,
989- struct work_struct *work)
990-{
991- int cpu, new_cpu;
992- 
993- cpu = new_cpu = smp_processor_id();
994- do {
995- new_cpu = cpumask_next(new_cpu, cpu_online_mask) % nr_cpu_ids;
996- if (cpu_to_node(new_cpu) == numa_node_id())
997- break;
998- } while (cpu != new_cpu);
999- 
1000- queue_work_on(new_cpu, wq, work);
1001-}
1002- 
1003/* This is called directly from KGD at ISR. */1120/* This is called directly from KGD at ISR. */
1004void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)1121void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
1005{1122{
@@ -1033,7 +1150,7 @@ void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
1033 patched_ihre, &is_patched)1150 patched_ihre, &is_patched)
1034 && enqueue_ih_ring_entry(node,1151 && enqueue_ih_ring_entry(node,
1035 is_patched ? patched_ihre : ih_ring_entry)) {1152 is_patched ? patched_ihre : ih_ring_entry)) {
1036- kfd_queue_work(node->ih_wq, &node->interrupt_work);1153+ queue_work(node->kfd->ih_wq, &node->interrupt_work);
1037 spin_unlock_irqrestore(&node->interrupt_lock, flags);1154 spin_unlock_irqrestore(&node->interrupt_lock, flags);
1038 return;1155 return;
1039 }1156 }
@@ -1330,6 +1447,13 @@ void kfd_dec_compute_active(struct kfd_node *node)
1330 WARN_ONCE(count < 0, "Compute profile ref. count error");1447 WARN_ONCE(count < 0, "Compute profile ref. count error");
1331}1448}
1332 1449 
1450+static bool kfd_compute_active(struct kfd_node *node)
1451+{
1452+ if (atomic_read(&node->kfd->compute_profile))
1453+ return true;
1454+ return false;
1455+}
1456+ 
1333void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint64_t throttle_bitmask)1457void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint64_t throttle_bitmask)
1334{1458{
1335 /*1459 /*
@@ -1363,25 +1487,223 @@ unsigned int kfd_get_num_xgmi_sdma_engines(struct kfd_node *node)
1363 kfd_get_num_sdma_engines(node);1487 kfd_get_num_sdma_engines(node);
1364}1488}
1365 1489 
1366-int kgd2kfd_check_and_lock_kfd(void)1490+int kgd2kfd_check_and_lock_kfd(struct kfd_dev *kfd)
1367{1491{
1492+ struct kfd_process *p;
1493+ int r = 0, temp, idx;
1494+ 
1368 mutex_lock(&kfd_processes_mutex);1495 mutex_lock(&kfd_processes_mutex);
1369- if (!hash_empty(kfd_processes_table) || kfd_is_locked()) {1496+ 
1370- mutex_unlock(&kfd_processes_mutex);1497+ /* kfd_processes_count is per kfd_dev, return -EBUSY without
1371- return -EBUSY;1498+ * further check
1499+ */
1500+ if (!!atomic_read(&kfd->kfd_processes_count)) {
1501+ pr_debug("process_wq_release not finished\n");
1502+ r = -EBUSY;
1503+ goto out;
1372 }1504 }
1373 1505 
1374- ++kfd_locked;1506+ if (hash_empty(kfd_processes_table) && !kfd_is_locked(kfd))
1507+ goto out;
1508+ 
1509+ /* fail under system reset/resume or kfd device is partition switching. */
1510+ if (kfd_is_locked(kfd)) {
1511+ r = -EBUSY;
1512+ goto out;
1513+ }
1514+ 
1515+ /*
1516+ * ensure all running processes are cgroup excluded from device before mode switch.
1517+ * i.e. no pdd was created on the process socket.
1518+ */
1519+ idx = srcu_read_lock(&kfd_processes_srcu);
1520+ hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1521+ int i;
1522+ 
1523+ for (i = 0; i < p->n_pdds; i++) {
1524+ if (p->pdds[i]->dev->kfd != kfd)
1525+ continue;
1526+ 
1527+ r = -EBUSY;
1528+ goto proc_check_unlock;
1529+ }
1530+ }
1531+ 
1532+proc_check_unlock:
1533+ srcu_read_unlock(&kfd_processes_srcu, idx);
1534+out:
1535+ if (!r)
1536+ ++kfd->kfd_dev_lock;
1375 mutex_unlock(&kfd_processes_mutex);1537 mutex_unlock(&kfd_processes_mutex);
1376 1538 
1539+ return r;
1540+}
1541+ 
1542+void kgd2kfd_unlock_kfd(struct kfd_dev *kfd)
1543+{
1544+ mutex_lock(&kfd_processes_mutex);
1545+ --kfd->kfd_dev_lock;
1546+ mutex_unlock(&kfd_processes_mutex);
1547+}
1548+ 
1549+int kgd2kfd_start_sched(struct kfd_dev *kfd, uint32_t node_id)
1550+{
1551+ struct kfd_node *node;
1552+ int ret;
1553+ 
1554+ if (!kfd->init_complete)
1555+ return 0;
1556+ 
1557+ if (node_id >= kfd->num_nodes) {
1558+ dev_warn(kfd->adev->dev, "Invalid node ID: %u exceeds %u\n",
1559+ node_id, kfd->num_nodes - 1);
1560+ return -EINVAL;
1561+ }
1562+ node = kfd->nodes[node_id];
1563+ 
1564+ ret = node->dqm->ops.unhalt(node->dqm);
1565+ if (ret)
1566+ dev_err(kfd_device, "Error in starting scheduler\n");
1567+ 
1568+ return ret;
1569+}
1570+ 
1571+int kgd2kfd_start_sched_all_nodes(struct kfd_dev *kfd)
1572+{
1573+ struct kfd_node *node;
1574+ int i, r;
1575+ 
1576+ if (!kfd->init_complete)
1577+ return 0;
1578+ 
1579+ for (i = 0; i < kfd->num_nodes; i++) {
1580+ node = kfd->nodes[i];
1581+ r = node->dqm->ops.unhalt(node->dqm);
1582+ if (r) {
1583+ dev_err(kfd_device, "Error in starting scheduler\n");
1584+ return r;
1585+ }
1586+ }
1377 return 0;1587 return 0;
1378}1588}
1379 1589 
1380-void kgd2kfd_unlock_kfd(void)1590+int kgd2kfd_stop_sched(struct kfd_dev *kfd, uint32_t node_id)
1381{1591{
1382- mutex_lock(&kfd_processes_mutex);1592+ struct kfd_node *node;
1383- --kfd_locked;1593+ 
1384- mutex_unlock(&kfd_processes_mutex);1594+ if (!kfd->init_complete)
1595+ return 0;
1596+ 
1597+ if (node_id >= kfd->num_nodes) {
1598+ dev_warn(kfd->adev->dev, "Invalid node ID: %u exceeds %u\n",
1599+ node_id, kfd->num_nodes - 1);
1600+ return -EINVAL;
1601+ }
1602+ 
1603+ node = kfd->nodes[node_id];
1604+ return node->dqm->ops.halt(node->dqm);
1605+}
1606+ 
1607+int kgd2kfd_stop_sched_all_nodes(struct kfd_dev *kfd)
1608+{
1609+ struct kfd_node *node;
1610+ int i, r;
1611+ 
1612+ if (!kfd->init_complete)
1613+ return 0;
1614+ 
1615+ for (i = 0; i < kfd->num_nodes; i++) {
1616+ node = kfd->nodes[i];
1617+ r = node->dqm->ops.halt(node->dqm);
1618+ if (r)
1619+ return r;
1620+ }
1621+ return 0;
1622+}
1623+ 
1624+bool kgd2kfd_compute_active(struct kfd_dev *kfd, uint32_t node_id)
1625+{
1626+ struct kfd_node *node;
1627+ 
1628+ if (!kfd->init_complete)
1629+ return false;
1630+ 
1631+ if (node_id >= kfd->num_nodes) {
1632+ dev_warn(kfd->adev->dev, "Invalid node ID: %u exceeds %u\n",
1633+ node_id, kfd->num_nodes - 1);
1634+ return false;
1635+ }
1636+ 
1637+ node = kfd->nodes[node_id];
1638+ 
1639+ return kfd_compute_active(node);
1640+}
1641+ 
1642+/**
1643+ * kgd2kfd_vmfault_fast_path() - KFD vm page fault interrupt handling fast path for gmc v9
1644+ * @adev: amdgpu device
1645+ * @entry: vm fault interrupt vector
1646+ * @retry_fault: if this is retry fault
1647+ *
1648+ * retry fault -
1649+ * with CAM enabled, adev primary ring
1650+ * | gmc_v9_0_process_interrupt()
1651+ * adev soft_ring
1652+ * | gmc_v9_0_process_interrupt() worker failed to recover page fault
1653+ * KFD node ih_fifo
1654+ * | KFD interrupt_wq worker
1655+ * kfd_signal_vm_fault_event
1656+ *
1657+ * without CAM, adev primary ring1
1658+ * | gmc_v9_0_process_interrupt worker failed to recvoer page fault
1659+ * KFD node ih_fifo
1660+ * | KFD interrupt_wq worker
1661+ * kfd_signal_vm_fault_event
1662+ *
1663+ * no-retry fault -
1664+ * adev primary ring
1665+ * | gmc_v9_0_process_interrupt()
1666+ * KFD node ih_fifo
1667+ * | KFD interrupt_wq worker
1668+ * kfd_signal_vm_fault_event
1669+ *
1670+ * fast path - After kfd_signal_vm_fault_event, gmc_v9_0_process_interrupt drop the page fault
1671+ * of same process, don't copy interrupt to KFD node ih_fifo.
1672+ * With gdb debugger enabled, need convert the retry fault to no-retry fault for
1673+ * debugger, cannot use the fast path.
1674+ *
1675+ * Return:
1676+ * true - use the fast path to handle this fault
1677+ * false - use normal path to handle it
1678+ */
1679+bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry,
1680+ bool retry_fault)
1681+{
1682+ struct kfd_process *p;
1683+ u32 cam_index;
1684+ 
1685+ if (entry->ih == &adev->irq.ih_soft || entry->ih == &adev->irq.ih1) {
1686+ p = kfd_lookup_process_by_pasid(entry->pasid, NULL);
1687+ if (!p)
1688+ return true;
1689+ 
1690+ if (p->gpu_page_fault && !p->debug_trap_enabled) {
1691+ if (retry_fault && adev->irq.retry_cam_enabled) {
1692+ cam_index = entry->src_data[2] & 0x3ff;
1693+ WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
1694+ }
1695+ 
1696+ kfd_unref_process(p);
1697+ return true;
1698+ }
1699+ 
1700+ /*
1701+ * This is the first page fault, set flag and then signal user space
1702+ */
1703+ p->gpu_page_fault = true;
1704+ kfd_unref_process(p);
1705+ }
1706+ return false;
1385}1707}
1386 1708 
1387#if defined(CONFIG_DEBUG_FS)1709#if defined(CONFIG_DEBUG_FS)
Mdrivers/gpu/drm/amd/amdkfd/kfd_priv.h+119-40
@@ -32,7 +32,7 @@
32#include <linux/atomic.h>32#include <linux/atomic.h>
33#include <linux/workqueue.h>33#include <linux/workqueue.h>
34#include <linux/spinlock.h>34#include <linux/spinlock.h>
35-#include <linux/kfd_ioctl.h>35+#include <uapi/linux/kfd_ioctl.h>
36#include <linux/idr.h>36#include <linux/idr.h>
37#include <linux/kfifo.h>37#include <linux/kfifo.h>
38#include <linux/seq_file.h>38#include <linux/seq_file.h>
@@ -99,11 +99,11 @@
99/*99/*
100 * Size of the per-process TBA+TMA buffer: 2 pages100 * Size of the per-process TBA+TMA buffer: 2 pages
101 *101 *
102- * The first page is the TBA used for the CWSR ISA code. The second102+ * The first chunk is the TBA used for the CWSR ISA code. The second
103- * page is used as TMA for user-mode trap handler setup in daisy-chain mode.103+ * chunk is used as TMA for user-mode trap handler setup in daisy-chain mode.
104 */104 */
105#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2)105#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2)
106-#define KFD_CWSR_TMA_OFFSET PAGE_SIZE106+#define KFD_CWSR_TMA_OFFSET (PAGE_SIZE + 2048)
107 107 
108#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \108#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \
109 (KFD_MAX_NUM_OF_PROCESSES * \109 (KFD_MAX_NUM_OF_PROCESSES * \
@@ -111,7 +111,14 @@
111 111 
112#define KFD_KERNEL_QUEUE_SIZE 2048112#define KFD_KERNEL_QUEUE_SIZE 2048
113 113 
114-#define KFD_UNMAP_LATENCY_MS (4000)114+/* KFD_UNMAP_LATENCY_MS is the timeout CP waiting for SDMA preemption. One XCC
115+ * can be associated to 2 SDMA engines. queue_preemption_timeout_ms is the time
116+ * driver waiting for CP returning the UNMAP_QUEUE fence. Thus the math is
117+ * queue_preemption_timeout_ms = sdma_preemption_time * 2 + cp workload
118+ * The format here makes CP workload 10% of total timeout
119+ */
120+#define KFD_UNMAP_LATENCY_MS \
121+ ((queue_preemption_timeout_ms - queue_preemption_timeout_ms / 10) >> 1)
115 122 
116#define KFD_MAX_SDMA_QUEUES 128123#define KFD_MAX_SDMA_QUEUES 128
117 124 
@@ -202,11 +209,13 @@ enum cache_policy {
202 cache_policy_noncoherent209 cache_policy_noncoherent
203};210};
204 211 
205-#define KFD_GC_VERSION(dev) ((dev)->adev->ip_versions[GC_HWIP][0])212+#define KFD_GC_VERSION(dev) (amdgpu_ip_version((dev)->adev, GC_HWIP, 0))
206#define KFD_IS_SOC15(dev) ((KFD_GC_VERSION(dev)) >= (IP_VERSION(9, 0, 1)))213#define KFD_IS_SOC15(dev) ((KFD_GC_VERSION(dev)) >= (IP_VERSION(9, 0, 1)))
207#define KFD_SUPPORT_XNACK_PER_PROCESS(dev)\214#define KFD_SUPPORT_XNACK_PER_PROCESS(dev)\
208 ((KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2)) || \215 ((KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2)) || \
209- (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3)))216+ (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3)) || \
217+ (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 4)) || \
218+ (KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0)))
210 219 
211struct kfd_node;220struct kfd_node;
212 221 
@@ -272,7 +281,6 @@ struct kfd_node {
272 281 
273 /* Interrupts */282 /* Interrupts */
274 struct kfifo ih_fifo;283 struct kfifo ih_fifo;
275- struct workqueue_struct *ih_wq;
276 struct work_struct interrupt_work;284 struct work_struct interrupt_work;
277 spinlock_t interrupt_lock;285 spinlock_t interrupt_lock;
278 286 
@@ -288,7 +296,6 @@ struct kfd_node {
288 296 
289 /* Global GWS resource shared between processes */297 /* Global GWS resource shared between processes */
290 void *gws;298 void *gws;
291- bool gws_debug_workaround;
292 299 
293 /* Clients watching SMI events */300 /* Clients watching SMI events */
294 struct list_head smi_clients;301 struct list_head smi_clients;
@@ -309,6 +316,10 @@ struct kfd_node {
309 struct kfd_local_mem_info local_mem_info;316 struct kfd_local_mem_info local_mem_info;
310 317 
311 struct kfd_dev *kfd;318 struct kfd_dev *kfd;
319+ 
320+ /* Track per device allocated watch points */
321+ uint32_t alloc_watch_ids;
322+ spinlock_t watch_points_lock;
312};323};
313 324 
314struct kfd_dev {325struct kfd_dev {
@@ -361,15 +372,18 @@ struct kfd_dev {
361 struct kfd_node *nodes[MAX_KFD_NODES];372 struct kfd_node *nodes[MAX_KFD_NODES];
362 unsigned int num_nodes;373 unsigned int num_nodes;
363 374 
364- /* Track per device allocated watch points */375+ struct workqueue_struct *ih_wq;
365- uint32_t alloc_watch_ids;
366- spinlock_t watch_points_lock;
367 376 
368 /* Kernel doorbells for KFD device */377 /* Kernel doorbells for KFD device */
369 struct amdgpu_bo *doorbells;378 struct amdgpu_bo *doorbells;
370 379 
371 /* bitmap for dynamic doorbell allocation from doorbell object */380 /* bitmap for dynamic doorbell allocation from doorbell object */
372 unsigned long *doorbell_bitmap;381 unsigned long *doorbell_bitmap;
382+ 
383+ /* for dynamic partitioning */
384+ int kfd_dev_lock;
385+ 
386+ atomic_t kfd_processes_count;
373};387};
374 388 
375enum kfd_mempool {389enum kfd_mempool {
@@ -413,13 +427,16 @@ enum kfd_unmap_queues_filter {
413 * @KFD_QUEUE_TYPE_DIQ: DIQ queue type.427 * @KFD_QUEUE_TYPE_DIQ: DIQ queue type.
414 *428 *
415 * @KFD_QUEUE_TYPE_SDMA_XGMI: Special SDMA queue for XGMI interface.429 * @KFD_QUEUE_TYPE_SDMA_XGMI: Special SDMA queue for XGMI interface.
430+ *
431+ * @KFD_QUEUE_TYPE_SDMA_BY_ENG_ID: SDMA user mode queue with target SDMA engine ID.
416 */432 */
417enum kfd_queue_type {433enum kfd_queue_type {
418 KFD_QUEUE_TYPE_COMPUTE,434 KFD_QUEUE_TYPE_COMPUTE,
419 KFD_QUEUE_TYPE_SDMA,435 KFD_QUEUE_TYPE_SDMA,
420 KFD_QUEUE_TYPE_HIQ,436 KFD_QUEUE_TYPE_HIQ,
421 KFD_QUEUE_TYPE_DIQ,437 KFD_QUEUE_TYPE_DIQ,
422- KFD_QUEUE_TYPE_SDMA_XGMI438+ KFD_QUEUE_TYPE_SDMA_XGMI,
439+ KFD_QUEUE_TYPE_SDMA_BY_ENG_ID
423};440};
424 441 
425enum kfd_queue_format {442enum kfd_queue_format {
@@ -493,8 +510,8 @@ struct queue_properties {
493 uint64_t queue_size;510 uint64_t queue_size;
494 uint32_t priority;511 uint32_t priority;
495 uint32_t queue_percent;512 uint32_t queue_percent;
496- uint32_t *read_ptr;513+ void __user *read_ptr;
497- uint32_t *write_ptr;514+ void __user *write_ptr;
498 void __iomem *doorbell_ptr;515 void __iomem *doorbell_ptr;
499 uint32_t doorbell_off;516 uint32_t doorbell_off;
500 bool is_interop;517 bool is_interop;
@@ -521,6 +538,12 @@ struct queue_properties {
521 uint64_t tba_addr;538 uint64_t tba_addr;
522 uint64_t tma_addr;539 uint64_t tma_addr;
523 uint64_t exception_status;540 uint64_t exception_status;
541+ 
542+ struct amdgpu_bo *wptr_bo;
543+ struct amdgpu_bo *rptr_bo;
544+ struct amdgpu_bo *ring_bo;
545+ struct amdgpu_bo *eop_buf_bo;
546+ struct amdgpu_bo *cwsr_bo;
524};547};
525 548 
526#define QUEUE_IS_ACTIVE(q) ((q).queue_size > 0 && \549#define QUEUE_IS_ACTIVE(q) ((q).queue_size > 0 && \
@@ -532,6 +555,7 @@ struct queue_properties {
532enum mqd_update_flag {555enum mqd_update_flag {
533 UPDATE_FLAG_DBG_WA_ENABLE = 1,556 UPDATE_FLAG_DBG_WA_ENABLE = 1,
534 UPDATE_FLAG_DBG_WA_DISABLE = 2,557 UPDATE_FLAG_DBG_WA_DISABLE = 2,
558+ UPDATE_FLAG_IS_GWS = 4, /* quirk for gfx9 IP */
535};559};
536 560 
537struct mqd_update_info {561struct mqd_update_info {
@@ -602,7 +626,7 @@ struct queue {
602 uint64_t gang_ctx_gpu_addr;626 uint64_t gang_ctx_gpu_addr;
603 void *gang_ctx_cpu_ptr;627 void *gang_ctx_cpu_ptr;
604 628 
605- struct amdgpu_bo *wptr_bo;629+ struct amdgpu_bo *wptr_bo_gart;
606};630};
607 631 
608enum KFD_MQD_TYPE {632enum KFD_MQD_TYPE {
@@ -748,7 +772,6 @@ struct kfd_process_device {
748 /* VM context for GPUVM allocations */772 /* VM context for GPUVM allocations */
749 struct file *drm_file;773 struct file *drm_file;
750 void *drm_priv;774 void *drm_priv;
751- atomic64_t tlb_seq;
752 775 
753 /* GPUVM allocations storage */776 /* GPUVM allocations storage */
754 struct idr alloc_idr;777 struct idr alloc_idr;
@@ -836,6 +859,11 @@ struct kfd_process_device {
836 void *proc_ctx_bo;859 void *proc_ctx_bo;
837 uint64_t proc_ctx_gpu_addr;860 uint64_t proc_ctx_gpu_addr;
838 void *proc_ctx_cpu_ptr;861 void *proc_ctx_cpu_ptr;
862+ 
863+ /* Tracks queue reset status */
864+ bool has_reset_queue;
865+ 
866+ u32 pasid;
839};867};
840 868 
841#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)869#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
@@ -853,6 +881,14 @@ struct svm_range_list {
853 struct delayed_work restore_work;881 struct delayed_work restore_work;
854 DECLARE_BITMAP(bitmap_supported, MAX_GPU_INSTANCE);882 DECLARE_BITMAP(bitmap_supported, MAX_GPU_INSTANCE);
855 struct task_struct *faulting_task;883 struct task_struct *faulting_task;
884+ /* check point ts decides if page fault recovery need be dropped */
885+ uint64_t checkpoint_ts[MAX_GPU_INSTANCE];
886+ 
887+ /* Default granularity to use in buffer migration
888+ * and restoration of backing memory while handling
889+ * recoverable page faults
890+ */
891+ uint8_t default_granularity;
856};892};
857 893 
858/* Process data */894/* Process data */
@@ -887,8 +923,6 @@ struct kfd_process {
887 /* We want to receive a notification when the mm_struct is destroyed */923 /* We want to receive a notification when the mm_struct is destroyed */
888 struct mmu_notifier mmu_notifier;924 struct mmu_notifier mmu_notifier;
889 925 
890- u32 pasid;
891- 
892 /*926 /*
893 * Array of kfd_process_device pointers,927 * Array of kfd_process_device pointers,
894 * one for each device the process is using.928 * one for each device the process is using.
@@ -918,7 +952,7 @@ struct kfd_process {
918 * fence will be triggered during eviction and new one will be created952 * fence will be triggered during eviction and new one will be created
919 * during restore953 * during restore
920 */954 */
921- struct dma_fence *ef;955+ struct dma_fence __rcu *ef;
922 956 
923 /* Work items for evicting and restoring BOs */957 /* Work items for evicting and restoring BOs */
924 struct delayed_work eviction_work;958 struct delayed_work eviction_work;
@@ -981,6 +1015,9 @@ struct kfd_process {
981 struct semaphore runtime_enable_sema;1015 struct semaphore runtime_enable_sema;
982 bool is_runtime_retry;1016 bool is_runtime_retry;
983 struct kfd_runtime_info runtime_info;1017 struct kfd_runtime_info runtime_info;
1018+ 
1019+ /* if gpu page fault sent to KFD */
1020+ bool gpu_page_fault;
984};1021};
985 1022 
986#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */1023#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
@@ -1013,7 +1050,8 @@ void kfd_process_destroy_wq(void);
1013void kfd_cleanup_processes(void);1050void kfd_cleanup_processes(void);
1014struct kfd_process *kfd_create_process(struct task_struct *thread);1051struct kfd_process *kfd_create_process(struct task_struct *thread);
1015struct kfd_process *kfd_get_process(const struct task_struct *task);1052struct kfd_process *kfd_get_process(const struct task_struct *task);
1016-struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid);1053+struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid,
1054+ struct kfd_process_device **pdd);
1017struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);1055struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
1018 1056 
1019int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id);1057int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id);
@@ -1065,8 +1103,6 @@ struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid);
1065/* PASIDs */1103/* PASIDs */
1066int kfd_pasid_init(void);1104int kfd_pasid_init(void);
1067void kfd_pasid_exit(void);1105void kfd_pasid_exit(void);
1068-bool kfd_set_pasid_limit(unsigned int new_limit);
1069-unsigned int kfd_get_pasid_limit(void);
1070u32 kfd_pasid_alloc(void);1106u32 kfd_pasid_alloc(void);
1071void kfd_pasid_free(u32 pasid);1107void kfd_pasid_free(u32 pasid);
1072 1108 
@@ -1116,7 +1152,6 @@ struct kfd_topology_device *kfd_topology_device_by_proximity_domain_no_lock(
1116 uint32_t proximity_domain);1152 uint32_t proximity_domain);
1117struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);1153struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);
1118struct kfd_node *kfd_device_by_id(uint32_t gpu_id);1154struct kfd_node *kfd_device_by_id(uint32_t gpu_id);
1119-struct kfd_node *kfd_device_by_pci_dev(const struct pci_dev *pdev);
1120static inline bool kfd_irq_is_from_node(struct kfd_node *node, uint32_t node_id,1155static inline bool kfd_irq_is_from_node(struct kfd_node *node, uint32_t node_id,
1121 uint32_t vmid)1156 uint32_t vmid)
1122{1157{
@@ -1128,7 +1163,9 @@ static inline struct kfd_node *kfd_node_by_irq_ids(struct amdgpu_device *adev,
1128 struct kfd_dev *dev = adev->kfd.dev;1163 struct kfd_dev *dev = adev->kfd.dev;
1129 uint32_t i;1164 uint32_t i;
1130 1165 
1131- if (KFD_GC_VERSION(dev) != IP_VERSION(9, 4, 3))1166+ if (KFD_GC_VERSION(dev) != IP_VERSION(9, 4, 3) &&
1167+ KFD_GC_VERSION(dev) != IP_VERSION(9, 4, 4) &&
1168+ KFD_GC_VERSION(dev) != IP_VERSION(9, 5, 0))
1132 return dev->nodes[0];1169 return dev->nodes[0];
1133 1170 
1134 for (i = 0; i < dev->num_nodes; i++)1171 for (i = 0; i < dev->num_nodes; i++)
@@ -1282,6 +1319,15 @@ int init_queue(struct queue **q, const struct queue_properties *properties);
1282void uninit_queue(struct queue *q);1319void uninit_queue(struct queue *q);
1283void print_queue_properties(struct queue_properties *q);1320void print_queue_properties(struct queue_properties *q);
1284void print_queue(struct queue *q);1321void print_queue(struct queue *q);
1322+int kfd_queue_buffer_get(struct amdgpu_vm *vm, void __user *addr, struct amdgpu_bo **pbo,
1323+ u64 expected_size);
1324+void kfd_queue_buffer_put(struct amdgpu_bo **bo);
1325+int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_properties *properties);
1326+int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_properties *properties);
1327+void kfd_queue_unref_bo_va(struct amdgpu_vm *vm, struct amdgpu_bo **bo);
1328+int kfd_queue_unref_bo_vas(struct kfd_process_device *pdd,
1329+ struct queue_properties *properties);
1330+void kfd_queue_ctx_save_restore_size(struct kfd_topology_device *dev);
1285 1331 
1286struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,1332struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
1287 struct kfd_node *dev);1333 struct kfd_node *dev);
@@ -1293,12 +1339,15 @@ struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
1293 struct kfd_node *dev);1339 struct kfd_node *dev);
1294struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type,1340struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type,
1295 struct kfd_node *dev);1341 struct kfd_node *dev);
1342+struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_TYPE type,
1343+ struct kfd_node *dev);
1296struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev);1344struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev);
1297void device_queue_manager_uninit(struct device_queue_manager *dqm);1345void device_queue_manager_uninit(struct device_queue_manager *dqm);
1298struct kernel_queue *kernel_queue_init(struct kfd_node *dev,1346struct kernel_queue *kernel_queue_init(struct kfd_node *dev,
1299 enum kfd_queue_type type);1347 enum kfd_queue_type type);
1300-void kernel_queue_uninit(struct kernel_queue *kq, bool hanging);1348+void kernel_queue_uninit(struct kernel_queue *kq);
1301-int kfd_dqm_evict_pasid(struct device_queue_manager *dqm, u32 pasid);1349+int kfd_evict_process_device(struct kfd_process_device *pdd);
1350+int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id);
1302 1351 
1303/* Process Queue Manager */1352/* Process Queue Manager */
1304struct process_queue_node {1353struct process_queue_node {
@@ -1313,10 +1362,8 @@ int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
1313void pqm_uninit(struct process_queue_manager *pqm);1362void pqm_uninit(struct process_queue_manager *pqm);
1314int pqm_create_queue(struct process_queue_manager *pqm,1363int pqm_create_queue(struct process_queue_manager *pqm,
1315 struct kfd_node *dev,1364 struct kfd_node *dev,
1316- struct file *f,
1317 struct queue_properties *properties,1365 struct queue_properties *properties,
1318 unsigned int *qid,1366 unsigned int *qid,
1319- struct amdgpu_bo *wptr_bo,
1320 const struct kfd_criu_queue_priv_data *q_data,1367 const struct kfd_criu_queue_priv_data *q_data,
1321 const void *restore_mqd,1368 const void *restore_mqd,
1322 const void *restore_ctl_stack,1369 const void *restore_ctl_stack,
@@ -1328,8 +1375,6 @@ int pqm_update_mqd(struct process_queue_manager *pqm, unsigned int qid,
1328 struct mqd_update_info *minfo);1375 struct mqd_update_info *minfo);
1329int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,1376int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
1330 void *gws);1377 void *gws);
1331-struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
1332- unsigned int qid);
1333struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,1378struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,
1334 unsigned int qid);1379 unsigned int qid);
1335int pqm_get_wave_state(struct process_queue_manager *pqm,1380int pqm_get_wave_state(struct process_queue_manager *pqm,
@@ -1343,7 +1388,7 @@ int pqm_get_queue_snapshot(struct process_queue_manager *pqm,
1343 int *num_qss_entries,1388 int *num_qss_entries,
1344 uint32_t *entry_size);1389 uint32_t *entry_size);
1345 1390 
1346-int amdkfd_fence_wait_timeout(uint64_t *fence_addr,1391+int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm,
1347 uint64_t fence_value,1392 uint64_t fence_value,
1348 unsigned int timeout_ms);1393 unsigned int timeout_ms);
1349 1394 
@@ -1356,6 +1401,24 @@ int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,
1356#define KFD_FENCE_COMPLETED (100)1401#define KFD_FENCE_COMPLETED (100)
1357#define KFD_FENCE_INIT (10)1402#define KFD_FENCE_INIT (10)
1358 1403 
1404+/**
1405+ * enum kfd_config_dequeue_wait_counts_cmd - Command for configuring
1406+ * dequeue wait counts.
1407+ *
1408+ * @KFD_DEQUEUE_WAIT_INIT: Set optimized dequeue wait counts for a
1409+ * certain ASICs. For these ASICs, this is default value used by RESET
1410+ * @KFD_DEQUEUE_WAIT_RESET: Reset dequeue wait counts to the optimized value
1411+ * for certain ASICs. For others set it to default hardware reset value
1412+ * @KFD_DEQUEUE_WAIT_SET_SCH_WAVE: Set context switch latency wait
1413+ *
1414+ */
1415+enum kfd_config_dequeue_wait_counts_cmd {
1416+ KFD_DEQUEUE_WAIT_INIT = 1,
1417+ KFD_DEQUEUE_WAIT_RESET = 2,
1418+ KFD_DEQUEUE_WAIT_SET_SCH_WAVE = 3
1419+};
1420+ 
1421+ 
1359struct packet_manager {1422struct packet_manager {
1360 struct device_queue_manager *dqm;1423 struct device_queue_manager *dqm;
1361 struct kernel_queue *priv_queue;1424 struct kernel_queue *priv_queue;
@@ -1381,8 +1444,8 @@ struct packet_manager_funcs {
1381 int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer,1444 int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer,
1382 enum kfd_unmap_queues_filter mode,1445 enum kfd_unmap_queues_filter mode,
1383 uint32_t filter_param, bool reset);1446 uint32_t filter_param, bool reset);
1384- int (*set_grace_period)(struct packet_manager *pm, uint32_t *buffer,1447+ int (*config_dequeue_wait_counts)(struct packet_manager *pm, uint32_t *buffer,
1385- uint32_t grace_period);1448+ enum kfd_config_dequeue_wait_counts_cmd cmd, uint32_t value);
1386 int (*query_status)(struct packet_manager *pm, uint32_t *buffer,1449 int (*query_status)(struct packet_manager *pm, uint32_t *buffer,
1387 uint64_t fence_address, uint64_t fence_value);1450 uint64_t fence_address, uint64_t fence_value);
1388 int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer);1451 int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer);
@@ -1393,7 +1456,7 @@ struct packet_manager_funcs {
1393 int set_resources_size;1456 int set_resources_size;
1394 int map_queues_size;1457 int map_queues_size;
1395 int unmap_queues_size;1458 int unmap_queues_size;
1396- int set_grace_period_size;1459+ int config_dequeue_wait_counts_size;
1397 int query_status_size;1460 int query_status_size;
1398 int release_mem_size;1461 int release_mem_size;
1399};1462};
@@ -1403,7 +1466,7 @@ extern const struct packet_manager_funcs kfd_v9_pm_funcs;
1403extern const struct packet_manager_funcs kfd_aldebaran_pm_funcs;1466extern const struct packet_manager_funcs kfd_aldebaran_pm_funcs;
1404 1467 
1405int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);1468int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
1406-void pm_uninit(struct packet_manager *pm, bool hanging);1469+void pm_uninit(struct packet_manager *pm);
1407int pm_send_set_resources(struct packet_manager *pm,1470int pm_send_set_resources(struct packet_manager *pm,
1408 struct scheduling_resources *res);1471 struct scheduling_resources *res);
1409int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues);1472int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues);
@@ -1416,7 +1479,9 @@ int pm_send_unmap_queue(struct packet_manager *pm,
1416 1479 
1417void pm_release_ib(struct packet_manager *pm);1480void pm_release_ib(struct packet_manager *pm);
1418 1481 
1419-int pm_update_grace_period(struct packet_manager *pm, uint32_t grace_period);1482+int pm_config_dequeue_wait_counts(struct packet_manager *pm,
1483+ enum kfd_config_dequeue_wait_counts_cmd cmd,
1484+ uint32_t wait_counts_config);
1420 1485 
1421/* Following PM funcs can be shared among VI and AI */1486/* Following PM funcs can be shared among VI and AI */
1422unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size);1487unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size);
@@ -1454,7 +1519,9 @@ int kfd_event_create(struct file *devkfd, struct kfd_process *p,
1454int kfd_get_num_events(struct kfd_process *p);1519int kfd_get_num_events(struct kfd_process *p);
1455int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);1520int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
1456 1521 
1457-void kfd_signal_vm_fault_event(struct kfd_node *dev, u32 pasid,1522+void kfd_signal_vm_fault_event_with_userptr(struct kfd_process *p, uint64_t gpu_va);
1523+ 
1524+void kfd_signal_vm_fault_event(struct kfd_process_device *pdd,
1458 struct kfd_vm_fault_info *info,1525 struct kfd_vm_fault_info *info,
1459 struct kfd_hsa_memory_exception_data *data);1526 struct kfd_hsa_memory_exception_data *data);
1460 1527 
@@ -1462,7 +1529,14 @@ void kfd_signal_reset_event(struct kfd_node *dev);
1462 1529 
1463void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid);1530void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid);
1464 1531 
1465-void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type);1532+static inline void kfd_flush_tlb(struct kfd_process_device *pdd,
1533+ enum TLB_FLUSH_TYPE type)
1534+{
1535+ struct amdgpu_device *adev = pdd->dev->adev;
1536+ struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv);
1537+ 
1538+ amdgpu_vm_flush_compute_tlb(adev, vm, type, pdd->dev->xcc_mask);
1539+}
1466 1540 
1467static inline bool kfd_flush_tlb_after_unmap(struct kfd_dev *dev)1541static inline bool kfd_flush_tlb_after_unmap(struct kfd_dev *dev)
1468{1542{
@@ -1474,7 +1548,7 @@ static inline bool kfd_flush_tlb_after_unmap(struct kfd_dev *dev)
1474int kfd_send_exception_to_runtime(struct kfd_process *p,1548int kfd_send_exception_to_runtime(struct kfd_process *p,
1475 unsigned int queue_id,1549 unsigned int queue_id,
1476 uint64_t error_reason);1550 uint64_t error_reason);
1477-bool kfd_is_locked(void);1551+bool kfd_is_locked(struct kfd_dev *kfd);
1478 1552 
1479/* Compute profile */1553/* Compute profile */
1480void kfd_inc_compute_active(struct kfd_node *dev);1554void kfd_inc_compute_active(struct kfd_node *dev);
@@ -1521,10 +1595,15 @@ int kfd_debugfs_hang_hws(struct kfd_node *dev);
1521int pm_debugfs_hang_hws(struct packet_manager *pm);1595int pm_debugfs_hang_hws(struct packet_manager *pm);
1522int dqm_debugfs_hang_hws(struct device_queue_manager *dqm);1596int dqm_debugfs_hang_hws(struct device_queue_manager *dqm);
1523 1597 
1598+void kfd_debugfs_add_process(struct kfd_process *p);
1599+void kfd_debugfs_remove_process(struct kfd_process *p);
1600+ 
1524#else1601#else
1525 1602 
1526static inline void kfd_debugfs_init(void) {}1603static inline void kfd_debugfs_init(void) {}
1527static inline void kfd_debugfs_fini(void) {}1604static inline void kfd_debugfs_fini(void) {}
1605+static inline void kfd_debugfs_add_process(struct kfd_process *p) {}
1606+static inline void kfd_debugfs_remove_process(struct kfd_process *p) {}
1528 1607 
1529#endif1608#endif
1530 1609 
Mdrivers/gpu/drm/amd/amdkfd/kfd_process.c+4-0
@@ -1061,6 +1061,8 @@ static void kfd_process_destroy_pdds(struct kfd_process *p)
1061 pdd->runtime_inuse = false;1061 pdd->runtime_inuse = false;
1062 }1062 }
1063 1063 
1064+ atomic_dec(&pdd->dev->kfd->kfd_processes_count);
1065+ 
1064 kfree(pdd);1066 kfree(pdd);
1065 p->pdds[i] = NULL;1067 p->pdds[i] = NULL;
1066 }1068 }
@@ -1623,6 +1625,8 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
1623 /* Init idr used for memory handle translation */1625 /* Init idr used for memory handle translation */
1624 idr_init(&pdd->alloc_idr);1626 idr_init(&pdd->alloc_idr);
1625 1627 
1628+ atomic_inc(&dev->kfd->kfd_processes_count);
1629+ 
1626 return pdd;1630 return pdd;
1627}1631}
1628 1632 
Mdrivers/hwtracing/coresight/coresight-tmc-etr.c+342-64
@@ -26,11 +26,19 @@ struct etr_flat_buf {
26 size_t size;26 size_t size;
27};27};
28 28 
29+struct etr_buf_hw {
30+ bool has_iommu;
31+ bool has_etr_sg;
32+ bool has_catu;
33+ bool has_resrv;
34+};
35+ 
29/*36/*
30 * etr_perf_buffer - Perf buffer used for ETR37 * etr_perf_buffer - Perf buffer used for ETR
31 * @drvdata - The ETR drvdaga this buffer has been allocated for.38 * @drvdata - The ETR drvdaga this buffer has been allocated for.
32 * @etr_buf - Actual buffer used by the ETR39 * @etr_buf - Actual buffer used by the ETR
33- * @pid - The PID this etr_perf_buffer belongs to.40+ * @pid - The PID of the session owner that etr_perf_buffer
41+ * belongs to.
34 * @snaphost - Perf session mode42 * @snaphost - Perf session mode
35 * @nr_pages - Number of pages in the ring buffer.43 * @nr_pages - Number of pages in the ring buffer.
36 * @pages - Array of Pages in the ring buffer.44 * @pages - Array of Pages in the ring buffer.
@@ -117,7 +125,7 @@ struct etr_sg_table {
117 * If we spill over to a new page for mapping 1 entry, we could as125 * If we spill over to a new page for mapping 1 entry, we could as
118 * well replace the link entry of the previous page with the last entry.126 * well replace the link entry of the previous page with the last entry.
119 */127 */
120-static inline unsigned long __attribute_const__128+static unsigned long __attribute_const__
121tmc_etr_sg_table_entries(int nr_pages)129tmc_etr_sg_table_entries(int nr_pages)
122{130{
123 unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;131 unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
@@ -231,13 +239,13 @@ static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
231 return -ENOMEM;239 return -ENOMEM;
232}240}
233 241 
234-static inline long242+static long
235tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)243tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
236{244{
237 return tmc_pages_get_offset(&sg_table->data_pages, addr);245 return tmc_pages_get_offset(&sg_table->data_pages, addr);
238}246}
239 247 
240-static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)248+static void tmc_free_table_pages(struct tmc_sg_table *sg_table)
241{249{
242 if (sg_table->table_vaddr)250 if (sg_table->table_vaddr)
243 vunmap(sg_table->table_vaddr);251 vunmap(sg_table->table_vaddr);
@@ -473,7 +481,7 @@ static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
473 dev_dbg(sg_table->dev, "******* End of Table *****\n");481 dev_dbg(sg_table->dev, "******* End of Table *****\n");
474}482}
475#else483#else
476-static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}484+static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
477#endif485#endif
478 486 
479/*487/*
@@ -688,6 +696,75 @@ static const struct etr_buf_operations etr_flat_buf_ops = {
688 .get_data = tmc_etr_get_data_flat_buf,696 .get_data = tmc_etr_get_data_flat_buf,
689};697};
690 698 
699+/*
700+ * tmc_etr_alloc_resrv_buf: Allocate a contiguous DMA buffer from reserved region.
701+ */
702+static int tmc_etr_alloc_resrv_buf(struct tmc_drvdata *drvdata,
703+ struct etr_buf *etr_buf, int node,
704+ void **pages)
705+{
706+ struct etr_flat_buf *resrv_buf;
707+ struct device *real_dev = drvdata->csdev->dev.parent;
708+ 
709+ /* We cannot reuse existing pages for resrv buf */
710+ if (pages)
711+ return -EINVAL;
712+ 
713+ resrv_buf = kzalloc(sizeof(*resrv_buf), GFP_KERNEL);
714+ if (!resrv_buf)
715+ return -ENOMEM;
716+ 
717+ resrv_buf->daddr = dma_map_resource(real_dev, drvdata->resrv_buf.paddr,
718+ drvdata->resrv_buf.size,
719+ DMA_FROM_DEVICE, 0);
720+ if (dma_mapping_error(real_dev, resrv_buf->daddr)) {
721+ dev_err(real_dev, "failed to map source buffer address\n");
722+ kfree(resrv_buf);
723+ return -ENOMEM;
724+ }
725+ 
726+ resrv_buf->vaddr = drvdata->resrv_buf.vaddr;
727+ resrv_buf->size = etr_buf->size = drvdata->resrv_buf.size;
728+ resrv_buf->dev = &drvdata->csdev->dev;
729+ etr_buf->hwaddr = resrv_buf->daddr;
730+ etr_buf->mode = ETR_MODE_RESRV;
731+ etr_buf->private = resrv_buf;
732+ return 0;
733+}
734+ 
735+static void tmc_etr_free_resrv_buf(struct etr_buf *etr_buf)
736+{
737+ struct etr_flat_buf *resrv_buf = etr_buf->private;
738+ 
739+ if (resrv_buf && resrv_buf->daddr) {
740+ struct device *real_dev = resrv_buf->dev->parent;
741+ 
742+ dma_unmap_resource(real_dev, resrv_buf->daddr,
743+ resrv_buf->size, DMA_FROM_DEVICE, 0);
744+ }
745+ kfree(resrv_buf);
746+}
747+ 
748+static void tmc_etr_sync_resrv_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
749+{
750+ /*
751+ * Adjust the buffer to point to the beginning of the trace data
752+ * and update the available trace data.
753+ */
754+ etr_buf->offset = rrp - etr_buf->hwaddr;
755+ if (etr_buf->full)
756+ etr_buf->len = etr_buf->size;
757+ else
758+ etr_buf->len = rwp - rrp;
759+}
760+ 
761+static const struct etr_buf_operations etr_resrv_buf_ops = {
762+ .alloc = tmc_etr_alloc_resrv_buf,
763+ .free = tmc_etr_free_resrv_buf,
764+ .sync = tmc_etr_sync_resrv_buf,
765+ .get_data = tmc_etr_get_data_flat_buf,
766+};
767+ 
691/*768/*
692 * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters769 * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
693 * appropriately.770 * appropriately.
@@ -794,6 +871,7 @@ static const struct etr_buf_operations *etr_buf_ops[] = {
794 [ETR_MODE_FLAT] = &etr_flat_buf_ops,871 [ETR_MODE_FLAT] = &etr_flat_buf_ops,
795 [ETR_MODE_ETR_SG] = &etr_sg_buf_ops,872 [ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
796 [ETR_MODE_CATU] = NULL,873 [ETR_MODE_CATU] = NULL,
874+ [ETR_MODE_RESRV] = &etr_resrv_buf_ops
797};875};
798 876 
799void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu)877void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu)
@@ -808,10 +886,8 @@ void tmc_etr_remove_catu_ops(void)
808}886}
809EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops);887EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops);
810 888 
811-static inline int tmc_etr_mode_alloc_buf(int mode,889+static int tmc_etr_mode_alloc_buf(int mode, struct tmc_drvdata *drvdata, struct etr_buf *etr_buf,
812- struct tmc_drvdata *drvdata,890+ int node, void **pages)
813- struct etr_buf *etr_buf, int node,
814- void **pages)
815{891{
816 int rc = -EINVAL;892 int rc = -EINVAL;
817 893 
@@ -819,6 +895,7 @@ static inline int tmc_etr_mode_alloc_buf(int mode,
819 case ETR_MODE_FLAT:895 case ETR_MODE_FLAT:
820 case ETR_MODE_ETR_SG:896 case ETR_MODE_ETR_SG:
821 case ETR_MODE_CATU:897 case ETR_MODE_CATU:
898+ case ETR_MODE_RESRV:
822 if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)899 if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
823 rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,900 rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
824 node, pages);901 node, pages);
@@ -830,6 +907,23 @@ static inline int tmc_etr_mode_alloc_buf(int mode,
830 }907 }
831}908}
832 909 
910+static void get_etr_buf_hw(struct device *dev, struct etr_buf_hw *buf_hw)
911+{
912+ struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
913+ 
914+ buf_hw->has_iommu = iommu_get_domain_for_dev(dev->parent);
915+ buf_hw->has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
916+ buf_hw->has_catu = !!tmc_etr_get_catu_device(drvdata);
917+ buf_hw->has_resrv = tmc_has_reserved_buffer(drvdata);
918+}
919+ 
920+static bool etr_can_use_flat_mode(struct etr_buf_hw *buf_hw, ssize_t etr_buf_size)
921+{
922+ bool has_sg = buf_hw->has_catu || buf_hw->has_etr_sg;
923+ 
924+ return !has_sg || buf_hw->has_iommu || etr_buf_size < SZ_1M;
925+}
926+ 
833/*927/*
834 * tmc_alloc_etr_buf: Allocate a buffer use by ETR.928 * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
835 * @drvdata : ETR device details.929 * @drvdata : ETR device details.
@@ -843,23 +937,22 @@ static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
843 int node, void **pages)937 int node, void **pages)
844{938{
845 int rc = -ENOMEM;939 int rc = -ENOMEM;
846- bool has_etr_sg, has_iommu;
847- bool has_sg, has_catu;
848 struct etr_buf *etr_buf;940 struct etr_buf *etr_buf;
941+ struct etr_buf_hw buf_hw;
849 struct device *dev = &drvdata->csdev->dev;942 struct device *dev = &drvdata->csdev->dev;
850 943 
851- has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);944+ get_etr_buf_hw(dev, &buf_hw);
852- has_iommu = iommu_get_domain_for_dev(dev->parent);
853- has_catu = !!tmc_etr_get_catu_device(drvdata);
854- 
855- has_sg = has_catu || has_etr_sg;
856- 
857 etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);945 etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
858 if (!etr_buf)946 if (!etr_buf)
859 return ERR_PTR(-ENOMEM);947 return ERR_PTR(-ENOMEM);
860 948 
861 etr_buf->size = size;949 etr_buf->size = size;
862 950 
951+ /* If there is user directive for buffer mode, try that first */
952+ if (drvdata->etr_mode != ETR_MODE_AUTO)
953+ rc = tmc_etr_mode_alloc_buf(drvdata->etr_mode, drvdata,
954+ etr_buf, node, pages);
955+ 
863 /*956 /*
864 * If we have to use an existing list of pages, we cannot reliably957 * If we have to use an existing list of pages, we cannot reliably
865 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,958 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
@@ -872,14 +965,13 @@ static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
872 * Fallback to available mechanisms.965 * Fallback to available mechanisms.
873 *966 *
874 */967 */
875- if (!pages &&968+ if (rc && !pages && etr_can_use_flat_mode(&buf_hw, size))
876- (!has_sg || has_iommu || size < SZ_1M))
877 rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,969 rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
878 etr_buf, node, pages);970 etr_buf, node, pages);
879- if (rc && has_etr_sg)971+ if (rc && buf_hw.has_etr_sg)
880 rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,972 rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
881 etr_buf, node, pages);973 etr_buf, node, pages);
882- if (rc && has_catu)974+ if (rc && buf_hw.has_catu)
883 rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,975 rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
884 etr_buf, node, pages);976 etr_buf, node, pages);
885 if (rc) {977 if (rc) {
@@ -915,7 +1007,7 @@ static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
915 return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);1007 return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
916}1008}
917 1009 
918-static inline s641010+static s64
919tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)1011tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
920{1012{
921 ssize_t len;1013 ssize_t len;
@@ -966,7 +1058,7 @@ static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
966 1058 
967static int __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)1059static int __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
968{1060{
969- u32 axictl, sts;1061+ u32 axictl, sts, ffcr;
970 struct etr_buf *etr_buf = drvdata->etr_buf;1062 struct etr_buf *etr_buf = drvdata->etr_buf;
971 int rc = 0;1063 int rc = 0;
972 1064 
@@ -1012,10 +1104,12 @@ static int __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
1012 writel_relaxed(sts, drvdata->base + TMC_STS);1104 writel_relaxed(sts, drvdata->base + TMC_STS);
1013 }1105 }
1014 1106 
1015- writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |1107+ ffcr = TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI | TMC_FFCR_FON_FLIN |
1016- TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |1108+ TMC_FFCR_FON_TRIG_EVT | TMC_FFCR_TRIGON_TRIGIN;
1017- TMC_FFCR_TRIGON_TRIGIN,1109+ if (drvdata->stop_on_flush)
1018- drvdata->base + TMC_FFCR);1110+ ffcr |= TMC_FFCR_STOP_ON_FLUSH;
1111+ writel_relaxed(ffcr, drvdata->base + TMC_FFCR);
1112+ 
1019 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);1113 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
1020 tmc_enable_hw(drvdata);1114 tmc_enable_hw(drvdata);
1021 1115 
@@ -1123,7 +1217,7 @@ static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1123 * When operating in sysFS mode the content of the buffer needs to be1217 * When operating in sysFS mode the content of the buffer needs to be
1124 * read before the TMC is disabled.1218 * read before the TMC is disabled.
1125 */1219 */
1126- if (drvdata->mode == CS_MODE_SYSFS)1220+ if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)
1127 tmc_etr_sync_sysfs_buf(drvdata);1221 tmc_etr_sync_sysfs_buf(drvdata);
1128 1222 
1129 tmc_disable_hw(drvdata);1223 tmc_disable_hw(drvdata);
@@ -1155,10 +1249,17 @@ static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev)
1155 * buffer, provided the size matches. Any allocation has to be done1249 * buffer, provided the size matches. Any allocation has to be done
1156 * with the lock released.1250 * with the lock released.
1157 */1251 */
1158- spin_lock_irqsave(&drvdata->spinlock, flags);1252+ raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1253+ 
1254+ /*
1255+ * If the ETR is already enabled, continue with the existing buffer.
1256+ */
1257+ if (coresight_get_mode(csdev) == CS_MODE_SYSFS)
1258+ goto out;
1259+ 
1159 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);1260 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1160 if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {1261 if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1161- spin_unlock_irqrestore(&drvdata->spinlock, flags);1262+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1162 1263 
1163 /* Allocate memory with the locks released */1264 /* Allocate memory with the locks released */
1164 free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);1265 free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
@@ -1166,10 +1267,10 @@ static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev)
1166 return new_buf;1267 return new_buf;
1167 1268 
1168 /* Let's try again */1269 /* Let's try again */
1169- spin_lock_irqsave(&drvdata->spinlock, flags);1270+ raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1170 }1271 }
1171 1272 
1172- if (drvdata->reading || drvdata->mode == CS_MODE_PERF) {1273+ if (drvdata->reading || coresight_get_mode(csdev) == CS_MODE_PERF) {
1173 ret = -EBUSY;1274 ret = -EBUSY;
1174 goto out;1275 goto out;
1175 }1276 }
@@ -1185,7 +1286,7 @@ static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev)
1185 }1286 }
1186 1287 
1187out:1288out:
1188- spin_unlock_irqrestore(&drvdata->spinlock, flags);1289+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1189 1290 
1190 /* Free memory outside the spinlock if need be */1291 /* Free memory outside the spinlock if need be */
1191 if (free_buf)1292 if (free_buf)
@@ -1203,26 +1304,26 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1203 if (IS_ERR(sysfs_buf))1304 if (IS_ERR(sysfs_buf))
1204 return PTR_ERR(sysfs_buf);1305 return PTR_ERR(sysfs_buf);
1205 1306 
1206- spin_lock_irqsave(&drvdata->spinlock, flags);1307+ raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1207 1308 
1208 /*1309 /*
1209 * In sysFS mode we can have multiple writers per sink. Since this1310 * In sysFS mode we can have multiple writers per sink. Since this
1210 * sink is already enabled no memory is needed and the HW need not be1311 * sink is already enabled no memory is needed and the HW need not be
1211 * touched, even if the buffer size has changed.1312 * touched, even if the buffer size has changed.
1212 */1313 */
1213- if (drvdata->mode == CS_MODE_SYSFS) {1314+ if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
1214- atomic_inc(&csdev->refcnt);1315+ csdev->refcnt++;
1215 goto out;1316 goto out;
1216 }1317 }
1217 1318 
1218 ret = tmc_etr_enable_hw(drvdata, sysfs_buf);1319 ret = tmc_etr_enable_hw(drvdata, sysfs_buf);
1219 if (!ret) {1320 if (!ret) {
1220- drvdata->mode = CS_MODE_SYSFS;1321+ coresight_set_mode(csdev, CS_MODE_SYSFS);
1221- atomic_inc(&csdev->refcnt);1322+ csdev->refcnt++;
1222 }1323 }
1223 1324 
1224out:1325out:
1225- spin_unlock_irqrestore(&drvdata->spinlock, flags);1326+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1226 1327 
1227 if (!ret)1328 if (!ret)
1228 dev_dbg(&csdev->dev, "TMC-ETR enabled\n");1329 dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
@@ -1540,18 +1641,19 @@ tmc_update_etr_buffer(struct coresight_device *csdev,
1540 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);1641 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1541 struct etr_perf_buffer *etr_perf = config;1642 struct etr_perf_buffer *etr_perf = config;
1542 struct etr_buf *etr_buf = etr_perf->etr_buf;1643 struct etr_buf *etr_buf = etr_perf->etr_buf;
1644+ struct perf_event *event = handle->event;
1543 1645 
1544- spin_lock_irqsave(&drvdata->spinlock, flags);1646+ raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1545 1647 
1546 /* Don't do anything if another tracer is using this sink */1648 /* Don't do anything if another tracer is using this sink */
1547- if (atomic_read(&csdev->refcnt) != 1) {1649+ if (csdev->refcnt != 1) {
1548- spin_unlock_irqrestore(&drvdata->spinlock, flags);1650+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1549 goto out;1651 goto out;
1550 }1652 }
1551 1653 
1552 if (WARN_ON(drvdata->perf_buf != etr_buf)) {1654 if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1553 lost = true;1655 lost = true;
1554- spin_unlock_irqrestore(&drvdata->spinlock, flags);1656+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1555 goto out;1657 goto out;
1556 }1658 }
1557 1659 
@@ -1561,7 +1663,7 @@ tmc_update_etr_buffer(struct coresight_device *csdev,
1561 tmc_sync_etr_buf(drvdata);1663 tmc_sync_etr_buf(drvdata);
1562 1664 
1563 CS_LOCK(drvdata->base);1665 CS_LOCK(drvdata->base);
1564- spin_unlock_irqrestore(&drvdata->spinlock, flags);1666+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1565 1667 
1566 lost = etr_buf->full;1668 lost = etr_buf->full;
1567 offset = etr_buf->offset;1669 offset = etr_buf->offset;
@@ -1609,6 +1711,15 @@ tmc_update_etr_buffer(struct coresight_device *csdev,
1609 */1711 */
1610 smp_wmb();1712 smp_wmb();
1611 1713 
1714+ /*
1715+ * If the event is active, it is triggered during an AUX pause.
1716+ * Re-enable the sink so that it is ready when AUX resume is invoked.
1717+ */
1718+ raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1719+ if (csdev->refcnt && !event->hw.state)
1720+ __tmc_etr_enable_hw(drvdata);
1721+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1722+ 
1612out:1723out:
1613 /*1724 /*
1614 * Don't set the TRUNCATED flag in snapshot mode because 1) the1725 * Don't set the TRUNCATED flag in snapshot mode because 1) the
@@ -1630,9 +1741,9 @@ static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1630 struct perf_output_handle *handle = data;1741 struct perf_output_handle *handle = data;
1631 struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);1742 struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1632 1743 
1633- spin_lock_irqsave(&drvdata->spinlock, flags);1744+ raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1634 /* Don't use this sink if it is already claimed by sysFS */1745 /* Don't use this sink if it is already claimed by sysFS */
1635- if (drvdata->mode == CS_MODE_SYSFS) {1746+ if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
1636 rc = -EBUSY;1747 rc = -EBUSY;
1637 goto unlock_out;1748 goto unlock_out;
1638 }1749 }
@@ -1642,7 +1753,7 @@ static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1642 goto unlock_out;1753 goto unlock_out;
1643 }1754 }
1644 1755 
1645- /* Get a handle on the pid of the process to monitor */1756+ /* Get a handle on the pid of the session owner */
1646 pid = etr_perf->pid;1757 pid = etr_perf->pid;
1647 1758 
1648 /* Do not proceed if this device is associated with another session */1759 /* Do not proceed if this device is associated with another session */
@@ -1656,7 +1767,7 @@ static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1656 * use for this session.1767 * use for this session.
1657 */1768 */
1658 if (drvdata->pid == pid) {1769 if (drvdata->pid == pid) {
1659- atomic_inc(&csdev->refcnt);1770+ csdev->refcnt++;
1660 goto unlock_out;1771 goto unlock_out;
1661 }1772 }
1662 1773 
@@ -1664,13 +1775,13 @@ static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1664 if (!rc) {1775 if (!rc) {
1665 /* Associate with monitored process. */1776 /* Associate with monitored process. */
1666 drvdata->pid = pid;1777 drvdata->pid = pid;
1667- drvdata->mode = CS_MODE_PERF;1778+ coresight_set_mode(csdev, CS_MODE_PERF);
1668 drvdata->perf_buf = etr_perf->etr_buf;1779 drvdata->perf_buf = etr_perf->etr_buf;
1669- atomic_inc(&csdev->refcnt);1780+ csdev->refcnt++;
1670 }1781 }
1671 1782 
1672unlock_out:1783unlock_out:
1673- spin_unlock_irqrestore(&drvdata->spinlock, flags);1784+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1674 return rc;1785 return rc;
1675}1786}
1676 1787 
@@ -1692,33 +1803,102 @@ static int tmc_disable_etr_sink(struct coresight_device *csdev)
1692 unsigned long flags;1803 unsigned long flags;
1693 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);1804 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1694 1805 
1695- spin_lock_irqsave(&drvdata->spinlock, flags);1806+ raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1696 1807 
1697 if (drvdata->reading) {1808 if (drvdata->reading) {
1698- spin_unlock_irqrestore(&drvdata->spinlock, flags);1809+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1699 return -EBUSY;1810 return -EBUSY;
1700 }1811 }
1701 1812 
1702- if (atomic_dec_return(&csdev->refcnt)) {1813+ csdev->refcnt--;
1703- spin_unlock_irqrestore(&drvdata->spinlock, flags);1814+ if (csdev->refcnt) {
1815+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1704 return -EBUSY;1816 return -EBUSY;
1705 }1817 }
1706 1818 
1707 /* Complain if we (somehow) got out of sync */1819 /* Complain if we (somehow) got out of sync */
1708- WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);1820+ WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED);
1709 tmc_etr_disable_hw(drvdata);1821 tmc_etr_disable_hw(drvdata);
1710 /* Dissociate from monitored process. */1822 /* Dissociate from monitored process. */
1711 drvdata->pid = -1;1823 drvdata->pid = -1;
1712- drvdata->mode = CS_MODE_DISABLED;1824+ coresight_set_mode(csdev, CS_MODE_DISABLED);
1713 /* Reset perf specific data */1825 /* Reset perf specific data */
1714 drvdata->perf_buf = NULL;1826 drvdata->perf_buf = NULL;
1715 1827 
1716- spin_unlock_irqrestore(&drvdata->spinlock, flags);1828+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1717 1829 
1718 dev_dbg(&csdev->dev, "TMC-ETR disabled\n");1830 dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1719 return 0;1831 return 0;
1720}1832}
1721 1833 
1834+static int tmc_panic_sync_etr(struct coresight_device *csdev)
1835+{
1836+ u32 val;
1837+ struct tmc_crash_metadata *mdata;
1838+ struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1839+ 
1840+ mdata = (struct tmc_crash_metadata *)drvdata->crash_mdata.vaddr;
1841+ 
1842+ if (!drvdata->etr_buf)
1843+ return 0;
1844+ 
1845+ /* Being in RESRV mode implies valid reserved memory as well */
1846+ if (drvdata->etr_buf->mode != ETR_MODE_RESRV)
1847+ return 0;
1848+ 
1849+ if (!tmc_has_crash_mdata_buffer(drvdata))
1850+ return 0;
1851+ 
1852+ CS_UNLOCK(drvdata->base);
1853+ 
1854+ /* Proceed only if ETR is enabled */
1855+ val = readl(drvdata->base + TMC_CTL);
1856+ if (!(val & TMC_CTL_CAPT_EN))
1857+ goto out;
1858+ 
1859+ val = readl(drvdata->base + TMC_FFSR);
1860+ /* Do manual flush and stop only if its not auto-stopped */
1861+ if (!(val & TMC_FFSR_FT_STOPPED)) {
1862+ dev_dbg(&csdev->dev,
1863+ "%s: Triggering manual flush\n", __func__);
1864+ tmc_flush_and_stop(drvdata);
1865+ } else
1866+ tmc_wait_for_tmcready(drvdata);
1867+ 
1868+ /* Sync registers from hardware to metadata region */
1869+ mdata->tmc_ram_size = readl(drvdata->base + TMC_RSZ);
1870+ mdata->tmc_sts = readl(drvdata->base + TMC_STS);
1871+ mdata->tmc_mode = readl(drvdata->base + TMC_MODE);
1872+ mdata->tmc_ffcr = readl(drvdata->base + TMC_FFCR);
1873+ mdata->tmc_ffsr = readl(drvdata->base + TMC_FFSR);
1874+ mdata->tmc_rrp = tmc_read_rrp(drvdata);
1875+ mdata->tmc_rwp = tmc_read_rwp(drvdata);
1876+ mdata->tmc_dba = tmc_read_dba(drvdata);
1877+ mdata->trace_paddr = drvdata->resrv_buf.paddr;
1878+ mdata->version = CS_CRASHDATA_VERSION;
1879+ 
1880+ /*
1881+ * Make sure all previous writes are ordered,
1882+ * before we mark valid
1883+ */
1884+ dmb(sy);
1885+ mdata->valid = true;
1886+ /*
1887+ * Below order need to maintained, since crc of metadata
1888+ * is dependent on first
1889+ */
1890+ mdata->crc32_tdata = find_crash_tracedata_crc(drvdata, mdata);
1891+ mdata->crc32_mdata = find_crash_metadata_crc(mdata);
1892+ 
1893+ tmc_disable_hw(drvdata);
1894+ 
1895+ dev_dbg(&csdev->dev, "%s: success\n", __func__);
1896+out:
1897+ CS_UNLOCK(drvdata->base);
1898+ 
1899+ return 0;
1900+}
1901+ 
1722static const struct coresight_ops_sink tmc_etr_sink_ops = {1902static const struct coresight_ops_sink tmc_etr_sink_ops = {
1723 .enable = tmc_enable_etr_sink,1903 .enable = tmc_enable_etr_sink,
1724 .disable = tmc_disable_etr_sink,1904 .disable = tmc_disable_etr_sink,
@@ -1727,8 +1907,13 @@ static const struct coresight_ops_sink tmc_etr_sink_ops = {
1727 .free_buffer = tmc_free_etr_buffer,1907 .free_buffer = tmc_free_etr_buffer,
1728};1908};
1729 1909 
1910+static const struct coresight_ops_panic tmc_etr_sync_ops = {
1911+ .sync = tmc_panic_sync_etr,
1912+};
1913+ 
1730const struct coresight_ops tmc_etr_cs_ops = {1914const struct coresight_ops tmc_etr_cs_ops = {
1731 .sink_ops = &tmc_etr_sink_ops,1915 .sink_ops = &tmc_etr_sink_ops,
1916+ .panic_ops = &tmc_etr_sync_ops,
1732};1917};
1733 1918 
1734int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)1919int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
@@ -1740,7 +1925,7 @@ int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1740 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))1925 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1741 return -EINVAL;1926 return -EINVAL;
1742 1927 
1743- spin_lock_irqsave(&drvdata->spinlock, flags);1928+ raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1744 if (drvdata->reading) {1929 if (drvdata->reading) {
1745 ret = -EBUSY;1930 ret = -EBUSY;
1746 goto out;1931 goto out;
@@ -1757,12 +1942,12 @@ int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1757 }1942 }
1758 1943 
1759 /* Disable the TMC if we are trying to read from a running session. */1944 /* Disable the TMC if we are trying to read from a running session. */
1760- if (drvdata->mode == CS_MODE_SYSFS)1945+ if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)
1761 __tmc_etr_disable_hw(drvdata);1946 __tmc_etr_disable_hw(drvdata);
1762 1947 
1763 drvdata->reading = true;1948 drvdata->reading = true;
1764out:1949out:
1765- spin_unlock_irqrestore(&drvdata->spinlock, flags);1950+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1766 1951 
1767 return ret;1952 return ret;
1768}1953}
@@ -1776,10 +1961,10 @@ int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1776 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))1961 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1777 return -EINVAL;1962 return -EINVAL;
1778 1963 
1779- spin_lock_irqsave(&drvdata->spinlock, flags);1964+ raw_spin_lock_irqsave(&drvdata->spinlock, flags);
1780 1965 
1781 /* RE-enable the TMC if need be */1966 /* RE-enable the TMC if need be */
1782- if (drvdata->mode == CS_MODE_SYSFS) {1967+ if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) {
1783 /*1968 /*
1784 * The trace run will continue with the same allocated trace1969 * The trace run will continue with the same allocated trace
1785 * buffer. Since the tracer is still enabled drvdata::buf can't1970 * buffer. Since the tracer is still enabled drvdata::buf can't
@@ -1796,7 +1981,7 @@ int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1796 }1981 }
1797 1982 
1798 drvdata->reading = false;1983 drvdata->reading = false;
1799- spin_unlock_irqrestore(&drvdata->spinlock, flags);1984+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
1800 1985 
1801 /* Free allocated memory out side of the spinlock */1986 /* Free allocated memory out side of the spinlock */
1802 if (sysfs_buf)1987 if (sysfs_buf)
@@ -1804,3 +1989,96 @@ int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1804 1989 
1805 return 0;1990 return 0;
1806}1991}
1992+ 
1993+static const char *const buf_modes_str[] = {
1994+ [ETR_MODE_FLAT] = "flat",
1995+ [ETR_MODE_ETR_SG] = "tmc-sg",
1996+ [ETR_MODE_CATU] = "catu",
1997+ [ETR_MODE_RESRV] = "resrv",
1998+ [ETR_MODE_AUTO] = "auto",
1999+};
2000+ 
2001+static ssize_t buf_modes_available_show(struct device *dev,
2002+ struct device_attribute *attr, char *buf)
2003+{
2004+ struct etr_buf_hw buf_hw;
2005+ ssize_t size = 0;
2006+ 
2007+ get_etr_buf_hw(dev, &buf_hw);
2008+ size += sysfs_emit(buf, "%s ", buf_modes_str[ETR_MODE_AUTO]);
2009+ size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_FLAT]);
2010+ if (buf_hw.has_etr_sg)
2011+ size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_ETR_SG]);
2012+ 
2013+ if (buf_hw.has_catu)
2014+ size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_CATU]);
2015+ 
2016+ if (buf_hw.has_resrv)
2017+ size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_RESRV]);
2018+ 
2019+ size += sysfs_emit_at(buf, size, "\n");
2020+ return size;
2021+}
2022+static DEVICE_ATTR_RO(buf_modes_available);
2023+ 
2024+static ssize_t buf_mode_preferred_show(struct device *dev,
2025+ struct device_attribute *attr, char *buf)
2026+{
2027+ struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
2028+ 
2029+ return sysfs_emit(buf, "%s\n", buf_modes_str[drvdata->etr_mode]);
2030+}
2031+ 
2032+static int buf_mode_set_resrv(struct tmc_drvdata *drvdata)
2033+{
2034+ int err = -EBUSY;
2035+ unsigned long flags;
2036+ struct tmc_resrv_buf *rbuf;
2037+ 
2038+ rbuf = &drvdata->resrv_buf;
2039+ 
2040+ /* Ensure there are no active crashdata read sessions */
2041+ raw_spin_lock_irqsave(&drvdata->spinlock, flags);
2042+ if (!rbuf->reading) {
2043+ tmc_crashdata_set_invalid(drvdata);
2044+ rbuf->len = 0;
2045+ drvdata->etr_mode = ETR_MODE_RESRV;
2046+ err = 0;
2047+ }
2048+ raw_spin_unlock_irqrestore(&drvdata->spinlock, flags);
2049+ return err;
2050+}
2051+ 
2052+static ssize_t buf_mode_preferred_store(struct device *dev,
2053+ struct device_attribute *attr,
2054+ const char *buf, size_t size)
2055+{
2056+ struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
2057+ struct etr_buf_hw buf_hw;
2058+ 
2059+ get_etr_buf_hw(dev, &buf_hw);
2060+ if (sysfs_streq(buf, buf_modes_str[ETR_MODE_FLAT]))
2061+ drvdata->etr_mode = ETR_MODE_FLAT;
2062+ else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_ETR_SG]) && buf_hw.has_etr_sg)
2063+ drvdata->etr_mode = ETR_MODE_ETR_SG;
2064+ else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_CATU]) && buf_hw.has_catu)
2065+ drvdata->etr_mode = ETR_MODE_CATU;
2066+ else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_RESRV]) && buf_hw.has_resrv)
2067+ return buf_mode_set_resrv(drvdata) ? : size;
2068+ else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_AUTO]))
2069+ drvdata->etr_mode = ETR_MODE_AUTO;
2070+ else
2071+ return -EINVAL;
2072+ return size;
2073+}
2074+static DEVICE_ATTR_RW(buf_mode_preferred);
2075+ 
2076+static struct attribute *coresight_etr_attrs[] = {
2077+ &dev_attr_buf_modes_available.attr,
2078+ &dev_attr_buf_mode_preferred.attr,
2079+ NULL,
2080+};
2081+ 
2082+const struct attribute_group coresight_etr_group = {
2083+ .attrs = coresight_etr_attrs,
2084+};
Mdrivers/net/can/usb/gs_usb.c+54-5
@@ -258,6 +258,11 @@ struct canfd_quirk {
258 u8 quirk;258 u8 quirk;
259} __packed;259} __packed;
260 260 
261+/* struct gs_host_frame::echo_id == GS_HOST_FRAME_ECHO_ID_RX indicates
262+ * a regular RX'ed CAN frame
263+ */
264+#define GS_HOST_FRAME_ECHO_ID_RX 0xffffffff
265+ 
261struct gs_host_frame {266struct gs_host_frame {
262 struct_group(header,267 struct_group(header,
263 u32 echo_id;268 u32 echo_id;
@@ -571,6 +576,37 @@ gs_usb_get_echo_skb(struct gs_can *dev, struct sk_buff *skb,
571 return len;576 return len;
572}577}
573 578 
579+static unsigned int
580+gs_usb_get_minimum_rx_length(const struct gs_can *dev, const struct gs_host_frame *hf,
581+ unsigned int *data_length_p)
582+{
583+ unsigned int minimum_length, data_length = 0;
584+ 
585+ if (hf->flags & GS_CAN_FLAG_FD) {
586+ if (hf->echo_id == GS_HOST_FRAME_ECHO_ID_RX)
587+ data_length = can_fd_dlc2len(hf->can_dlc);
588+ 
589+ if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP)
590+ /* timestamp follows data field of max size */
591+ minimum_length = struct_size(hf, canfd_ts, 1);
592+ else
593+ minimum_length = sizeof(hf->header) + data_length;
594+ } else {
595+ if (hf->echo_id == GS_HOST_FRAME_ECHO_ID_RX &&
596+ !(hf->can_id & cpu_to_le32(CAN_RTR_FLAG)))
597+ data_length = can_cc_dlc2len(hf->can_dlc);
598+ 
599+ if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP)
600+ /* timestamp follows data field of max size */
601+ minimum_length = struct_size(hf, classic_can_ts, 1);
602+ else
603+ minimum_length = sizeof(hf->header) + data_length;
604+ }
605+ 
606+ *data_length_p = data_length;
607+ return minimum_length;
608+}
609+ 
574static void gs_usb_receive_bulk_callback(struct urb *urb)610static void gs_usb_receive_bulk_callback(struct urb *urb)
575{611{
576 struct gs_usb *parent = urb->context;612 struct gs_usb *parent = urb->context;
@@ -579,7 +615,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb)
579 int rc;615 int rc;
580 struct net_device_stats *stats;616 struct net_device_stats *stats;
581 struct gs_host_frame *hf = urb->transfer_buffer;617 struct gs_host_frame *hf = urb->transfer_buffer;
582- unsigned int minimum_length;618+ unsigned int minimum_length, data_length;
583 struct gs_tx_context *txc;619 struct gs_tx_context *txc;
584 struct can_frame *cf;620 struct can_frame *cf;
585 struct canfd_frame *cfd;621 struct canfd_frame *cfd;
@@ -622,20 +658,33 @@ static void gs_usb_receive_bulk_callback(struct urb *urb)
622 if (!netif_running(netdev))658 if (!netif_running(netdev))
623 goto resubmit_urb;659 goto resubmit_urb;
624 660 
625- if (hf->echo_id == -1) { /* normal rx */661+ minimum_length = gs_usb_get_minimum_rx_length(dev, hf, &data_length);
662+ if (urb->actual_length < minimum_length) {
663+ stats->rx_errors++;
664+ stats->rx_length_errors++;
665+ 
666+ if (net_ratelimit())
667+ netdev_err(netdev,
668+ "short read (actual_length=%u, minimum_length=%u)\n",
669+ urb->actual_length, minimum_length);
670+ 
671+ goto resubmit_urb;
672+ }
673+ 
674+ if (hf->echo_id == GS_HOST_FRAME_ECHO_ID_RX) { /* normal rx */
626 if (hf->flags & GS_CAN_FLAG_FD) {675 if (hf->flags & GS_CAN_FLAG_FD) {
627 skb = alloc_canfd_skb(netdev, &cfd);676 skb = alloc_canfd_skb(netdev, &cfd);
628 if (!skb)677 if (!skb)
629 return;678 return;
630 679 
631 cfd->can_id = le32_to_cpu(hf->can_id);680 cfd->can_id = le32_to_cpu(hf->can_id);
632- cfd->len = can_fd_dlc2len(hf->can_dlc);681+ cfd->len = data_length;
633 if (hf->flags & GS_CAN_FLAG_BRS)682 if (hf->flags & GS_CAN_FLAG_BRS)
634 cfd->flags |= CANFD_BRS;683 cfd->flags |= CANFD_BRS;
635 if (hf->flags & GS_CAN_FLAG_ESI)684 if (hf->flags & GS_CAN_FLAG_ESI)
636 cfd->flags |= CANFD_ESI;685 cfd->flags |= CANFD_ESI;
637 686 
638- memcpy(cfd->data, hf->canfd->data, cfd->len);687+ memcpy(cfd->data, hf->canfd->data, data_length);
639 } else {688 } else {
640 skb = alloc_can_skb(netdev, &cf);689 skb = alloc_can_skb(netdev, &cf);
641 if (!skb)690 if (!skb)
@@ -644,7 +693,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb)
644 cf->can_id = le32_to_cpu(hf->can_id);693 cf->can_id = le32_to_cpu(hf->can_id);
645 can_frame_set_cc_len(cf, hf->can_dlc, dev->can.ctrlmode);694 can_frame_set_cc_len(cf, hf->can_dlc, dev->can.ctrlmode);
646 695 
647- memcpy(cf->data, hf->classic_can->data, 8);696+ memcpy(cf->data, hf->classic_can->data, data_length);
648 697 
649 /* ERROR frames tell us information about the controller */698 /* ERROR frames tell us information about the controller */
650 if (le32_to_cpu(hf->can_id) & CAN_ERR_FLAG)699 if (le32_to_cpu(hf->can_id) & CAN_ERR_FLAG)
Mdrivers/platform/x86/amd/pmc/pmc.c+119-373
@@ -10,8 +10,8 @@
10 10 
11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12 12 
13-#include <asm/amd_nb.h>
14#include <linux/acpi.h>13#include <linux/acpi.h>
14+#include <linux/array_size.h>
15#include <linux/bitfield.h>15#include <linux/bitfield.h>
16#include <linux/bits.h>16#include <linux/bits.h>
17#include <linux/debugfs.h>17#include <linux/debugfs.h>
@@ -28,90 +28,36 @@
28#include <linux/seq_file.h>28#include <linux/seq_file.h>
29#include <linux/uaccess.h>29#include <linux/uaccess.h>
30 30 
31+#include <asm/amd/node.h>
32+ 
31#include "pmc.h"33#include "pmc.h"
32 34 
33-/* SMU communication registers */35+static const struct amd_pmc_bit_map soc15_ip_blk_v2[] = {
34-#define AMD_PMC_REGISTER_MESSAGE 0x53836+ {"DISPLAY", BIT(0)},
35-#define AMD_PMC_REGISTER_RESPONSE 0x98037+ {"CPU", BIT(1)},
36-#define AMD_PMC_REGISTER_ARGUMENT 0x9BC38+ {"GFX", BIT(2)},
37- 39+ {"VDD", BIT(3)},
38-/* PMC Scratch Registers */40+ {"VDD_CCX", BIT(4)},
39-#define AMD_PMC_SCRATCH_REG_CZN 0x9441+ {"ACP", BIT(5)},
40-#define AMD_PMC_SCRATCH_REG_YC 0xD1442+ {"VCN_0", BIT(6)},
41- 43+ {"VCN_1", BIT(7)},
42-/* STB Registers */44+ {"ISP", BIT(8)},
43-#define AMD_PMC_STB_PMI_0 0x03E3060045+ {"NBIO", BIT(9)},
44-#define AMD_PMC_STB_S2IDLE_PREPARE 0xC600000146+ {"DF", BIT(10)},
45-#define AMD_PMC_STB_S2IDLE_RESTORE 0xC600000247+ {"USB3_0", BIT(11)},
46-#define AMD_PMC_STB_S2IDLE_CHECK 0xC600000348+ {"USB3_1", BIT(12)},
47-#define AMD_PMC_STB_DUMMY_PC 0xC600000749+ {"LAPIC", BIT(13)},
48- 50+ {"USB3_2", BIT(14)},
49-/* STB S2D(Spill to DRAM) has different message port offset */51+ {"USB4_RT0", BIT(15)},
50-#define AMD_S2D_REGISTER_MESSAGE 0xA2052+ {"USB4_RT1", BIT(16)},
51-#define AMD_S2D_REGISTER_RESPONSE 0xA8053+ {"USB4_0", BIT(17)},
52-#define AMD_S2D_REGISTER_ARGUMENT 0xA8854+ {"USB4_1", BIT(18)},
53- 55+ {"MPM", BIT(19)},
54-/* STB Spill to DRAM Parameters */56+ {"JPEG_0", BIT(20)},
55-#define S2D_TELEMETRY_BYTES_MAX 0x10000057+ {"JPEG_1", BIT(21)},
56-#define S2D_TELEMETRY_DRAMBYTES_MAX 0x100000058+ {"IPU", BIT(22)},
57- 59+ {"UMSCH", BIT(23)},
58-/* Base address of SMU for mapping physical address to virtual address */60+ {"VPE", BIT(24)},
59-#define AMD_PMC_MAPPING_SIZE 0x01000
60-#define AMD_PMC_BASE_ADDR_OFFSET 0x10000
61-#define AMD_PMC_BASE_ADDR_LO 0x13B102E8
62-#define AMD_PMC_BASE_ADDR_HI 0x13B102EC
63-#define AMD_PMC_BASE_ADDR_LO_MASK GENMASK(15, 0)
64-#define AMD_PMC_BASE_ADDR_HI_MASK GENMASK(31, 20)
65- 
66-/* SMU Response Codes */
67-#define AMD_PMC_RESULT_OK 0x01
68-#define AMD_PMC_RESULT_CMD_REJECT_BUSY 0xFC
69-#define AMD_PMC_RESULT_CMD_REJECT_PREREQ 0xFD
70-#define AMD_PMC_RESULT_CMD_UNKNOWN 0xFE
71-#define AMD_PMC_RESULT_FAILED 0xFF
72- 
73-/* FCH SSC Registers */
74-#define FCH_S0I3_ENTRY_TIME_L_OFFSET 0x30
75-#define FCH_S0I3_ENTRY_TIME_H_OFFSET 0x34
76-#define FCH_S0I3_EXIT_TIME_L_OFFSET 0x38
77-#define FCH_S0I3_EXIT_TIME_H_OFFSET 0x3C
78-#define FCH_SSC_MAPPING_SIZE 0x800
79-#define FCH_BASE_PHY_ADDR_LOW 0xFED81100
80-#define FCH_BASE_PHY_ADDR_HIGH 0x00000000
81- 
82-/* SMU Message Definations */
83-#define SMU_MSG_GETSMUVERSION 0x02
84-#define SMU_MSG_LOG_GETDRAM_ADDR_HI 0x04
85-#define SMU_MSG_LOG_GETDRAM_ADDR_LO 0x05
86-#define SMU_MSG_LOG_START 0x06
87-#define SMU_MSG_LOG_RESET 0x07
88-#define SMU_MSG_LOG_DUMP_DATA 0x08
89-#define SMU_MSG_GET_SUP_CONSTRAINTS 0x09
90- 
91-#define PMC_MSG_DELAY_MIN_US 50
92-#define RESPONSE_REGISTER_LOOP_MAX 20000
93- 
94-#define DELAY_MIN_US 2000
95-#define DELAY_MAX_US 3000
96-#define FIFO_SIZE 4096
97- 
98-enum amd_pmc_def {
99- MSG_TEST = 0x01,
100- MSG_OS_HINT_PCO,
101- MSG_OS_HINT_RN,
102-};
103- 
104-enum s2d_arg {
105- S2D_TELEMETRY_SIZE = 0x01,
106- S2D_PHYS_ADDR_LOW,
107- S2D_PHYS_ADDR_HIGH,
108- S2D_NUM_SAMPLES,
109- S2D_DRAM_SIZE,
110-};
111- 
112-struct amd_pmc_bit_map {
113- const char *name;
114- u32 bit_mask;
115};61};
116 62 
117static const struct amd_pmc_bit_map soc15_ip_blk[] = {63static const struct amd_pmc_bit_map soc15_ip_blk[] = {
@@ -136,21 +82,14 @@ static const struct amd_pmc_bit_map soc15_ip_blk[] = {
136 {"JPEG", BIT(18)},82 {"JPEG", BIT(18)},
137 {"IPU", BIT(19)},83 {"IPU", BIT(19)},
138 {"UMSCH", BIT(20)},84 {"UMSCH", BIT(20)},
139- {}85+ {"VPE", BIT(21)},
140};86};
141 87 
142-static bool enable_stb;
143-module_param(enable_stb, bool, 0644);
144-MODULE_PARM_DESC(enable_stb, "Enable the STB debug mechanism");
145- 
146static bool disable_workarounds;88static bool disable_workarounds;
147module_param(disable_workarounds, bool, 0644);89module_param(disable_workarounds, bool, 0644);
148MODULE_PARM_DESC(disable_workarounds, "Disable workarounds for platform bugs");90MODULE_PARM_DESC(disable_workarounds, "Disable workarounds for platform bugs");
149 91 
150static struct amd_pmc_dev pmc;92static struct amd_pmc_dev pmc;
151-static int amd_pmc_send_cmd(struct amd_pmc_dev *dev, u32 arg, u32 *data, u8 msg, bool ret);
152-static int amd_pmc_read_stb(struct amd_pmc_dev *dev, u32 *buf);
153-static int amd_pmc_write_stb(struct amd_pmc_dev *dev, u32 data);
154 93 
155static inline u32 amd_pmc_reg_read(struct amd_pmc_dev *dev, int reg_offset)94static inline u32 amd_pmc_reg_read(struct amd_pmc_dev *dev, int reg_offset)
156{95{
@@ -162,146 +101,33 @@ static inline void amd_pmc_reg_write(struct amd_pmc_dev *dev, int reg_offset, u3
162 iowrite32(val, dev->regbase + reg_offset);101 iowrite32(val, dev->regbase + reg_offset);
163}102}
164 103 
165-struct smu_metrics {
166- u32 table_version;
167- u32 hint_count;
168- u32 s0i3_last_entry_status;
169- u32 timein_s0i2;
170- u64 timeentering_s0i3_lastcapture;
171- u64 timeentering_s0i3_totaltime;
172- u64 timeto_resume_to_os_lastcapture;
173- u64 timeto_resume_to_os_totaltime;
174- u64 timein_s0i3_lastcapture;
175- u64 timein_s0i3_totaltime;
176- u64 timein_swdrips_lastcapture;
177- u64 timein_swdrips_totaltime;
178- u64 timecondition_notmet_lastcapture[32];
179- u64 timecondition_notmet_totaltime[32];
180-} __packed;
181- 
182-static int amd_pmc_stb_debugfs_open(struct inode *inode, struct file *filp)
183-{
184- struct amd_pmc_dev *dev = filp->f_inode->i_private;
185- u32 size = FIFO_SIZE * sizeof(u32);
186- u32 *buf;
187- int rc;
188- 
189- buf = kzalloc(size, GFP_KERNEL);
190- if (!buf)
191- return -ENOMEM;
192- 
193- rc = amd_pmc_read_stb(dev, buf);
194- if (rc) {
195- kfree(buf);
196- return rc;
197- }
198- 
199- filp->private_data = buf;
200- return rc;
201-}
202- 
203-static ssize_t amd_pmc_stb_debugfs_read(struct file *filp, char __user *buf, size_t size,
204- loff_t *pos)
205-{
206- if (!filp->private_data)
207- return -EINVAL;
208- 
209- return simple_read_from_buffer(buf, size, pos, filp->private_data,
210- FIFO_SIZE * sizeof(u32));
211-}
212- 
213-static int amd_pmc_stb_debugfs_release(struct inode *inode, struct file *filp)
214-{
215- kfree(filp->private_data);
216- return 0;
217-}
218- 
219-static const struct file_operations amd_pmc_stb_debugfs_fops = {
220- .owner = THIS_MODULE,
221- .open = amd_pmc_stb_debugfs_open,
222- .read = amd_pmc_stb_debugfs_read,
223- .release = amd_pmc_stb_debugfs_release,
224-};
225- 
226-static int amd_pmc_stb_debugfs_open_v2(struct inode *inode, struct file *filp)
227-{
228- struct amd_pmc_dev *dev = filp->f_inode->i_private;
229- u32 *buf, fsize, num_samples, stb_rdptr_offset = 0;
230- int ret;
231- 
232- /* Write dummy postcode while reading the STB buffer */
233- ret = amd_pmc_write_stb(dev, AMD_PMC_STB_DUMMY_PC);
234- if (ret)
235- dev_err(dev->dev, "error writing to STB: %d\n", ret);
236- 
237- buf = kzalloc(S2D_TELEMETRY_BYTES_MAX, GFP_KERNEL);
238- if (!buf)
239- return -ENOMEM;
240- 
241- /* Spill to DRAM num_samples uses separate SMU message port */
242- dev->msg_port = 1;
243- 
244- /* Get the num_samples to calculate the last push location */
245- ret = amd_pmc_send_cmd(dev, S2D_NUM_SAMPLES, &num_samples, dev->s2d_msg_id, true);
246- /* Clear msg_port for other SMU operation */
247- dev->msg_port = 0;
248- if (ret) {
249- dev_err(dev->dev, "error: S2D_NUM_SAMPLES not supported : %d\n", ret);
250- kfree(buf);
251- return ret;
252- }
253- 
254- /* Start capturing data from the last push location */
255- if (num_samples > S2D_TELEMETRY_BYTES_MAX) {
256- fsize = S2D_TELEMETRY_BYTES_MAX;
257- stb_rdptr_offset = num_samples - fsize;
258- } else {
259- fsize = num_samples;
260- stb_rdptr_offset = 0;
261- }
262- 
263- memcpy_fromio(buf, dev->stb_virt_addr + stb_rdptr_offset, fsize);
264- filp->private_data = buf;
265- 
266- return 0;
267-}
268- 
269-static ssize_t amd_pmc_stb_debugfs_read_v2(struct file *filp, char __user *buf, size_t size,
270- loff_t *pos)
271-{
272- if (!filp->private_data)
273- return -EINVAL;
274- 
275- return simple_read_from_buffer(buf, size, pos, filp->private_data,
276- S2D_TELEMETRY_BYTES_MAX);
277-}
278- 
279-static int amd_pmc_stb_debugfs_release_v2(struct inode *inode, struct file *filp)
280-{
281- kfree(filp->private_data);
282- return 0;
283-}
284- 
285-static const struct file_operations amd_pmc_stb_debugfs_fops_v2 = {
286- .owner = THIS_MODULE,
287- .open = amd_pmc_stb_debugfs_open_v2,
288- .read = amd_pmc_stb_debugfs_read_v2,
289- .release = amd_pmc_stb_debugfs_release_v2,
290-};
291- 
292static void amd_pmc_get_ip_info(struct amd_pmc_dev *dev)104static void amd_pmc_get_ip_info(struct amd_pmc_dev *dev)
293{105{
294 switch (dev->cpu_id) {106 switch (dev->cpu_id) {
295 case AMD_CPU_ID_PCO:107 case AMD_CPU_ID_PCO:
296 case AMD_CPU_ID_RN:108 case AMD_CPU_ID_RN:
109+ case AMD_CPU_ID_VG:
297 case AMD_CPU_ID_YC:110 case AMD_CPU_ID_YC:
298 case AMD_CPU_ID_CB:111 case AMD_CPU_ID_CB:
299 dev->num_ips = 12;112 dev->num_ips = 12;
300- dev->s2d_msg_id = 0xBE;113+ dev->ips_ptr = soc15_ip_blk;
114+ dev->smu_msg = 0x538;
301 break;115 break;
302 case AMD_CPU_ID_PS:116 case AMD_CPU_ID_PS:
303 dev->num_ips = 21;117 dev->num_ips = 21;
304- dev->s2d_msg_id = 0x85;118+ dev->ips_ptr = soc15_ip_blk;
119+ dev->smu_msg = 0x538;
120+ break;
121+ case PCI_DEVICE_ID_AMD_1AH_M20H_ROOT:
122+ case PCI_DEVICE_ID_AMD_1AH_M60H_ROOT:
123+ if (boot_cpu_data.x86_model == 0x70) {
124+ dev->num_ips = ARRAY_SIZE(soc15_ip_blk_v2);
125+ dev->ips_ptr = soc15_ip_blk_v2;
126+ } else {
127+ dev->num_ips = ARRAY_SIZE(soc15_ip_blk);
128+ dev->ips_ptr = soc15_ip_blk;
129+ }
130+ dev->smu_msg = 0x938;
305 break;131 break;
306 }132 }
307}133}
@@ -343,11 +169,12 @@ static int amd_pmc_setup_smu_logging(struct amd_pmc_dev *dev)
343 169 
344static int get_metrics_table(struct amd_pmc_dev *pdev, struct smu_metrics *table)170static int get_metrics_table(struct amd_pmc_dev *pdev, struct smu_metrics *table)
345{171{
346- if (!pdev->smu_virt_addr) {172+ int rc;
347- int ret = amd_pmc_setup_smu_logging(pdev);
348 173 
349- if (ret)174+ if (!pdev->smu_virt_addr) {
350- return ret;175+ rc = amd_pmc_setup_smu_logging(pdev);
176+ if (rc)
177+ return rc;
351 }178 }
352 179 
353 if (pdev->cpu_id == AMD_CPU_ID_PCO)180 if (pdev->cpu_id == AMD_CPU_ID_PCO)
@@ -396,10 +223,10 @@ static ssize_t smu_fw_version_show(struct device *d, struct device_attribute *at
396 char *buf)223 char *buf)
397{224{
398 struct amd_pmc_dev *dev = dev_get_drvdata(d);225 struct amd_pmc_dev *dev = dev_get_drvdata(d);
226+ int rc;
399 227 
400 if (!dev->major) {228 if (!dev->major) {
401- int rc = amd_pmc_get_smu_version(dev);229+ rc = amd_pmc_get_smu_version(dev);
402- 
403 if (rc)230 if (rc)
404 return rc;231 return rc;
405 }232 }
@@ -410,10 +237,10 @@ static ssize_t smu_program_show(struct device *d, struct device_attribute *attr,
410 char *buf)237 char *buf)
411{238{
412 struct amd_pmc_dev *dev = dev_get_drvdata(d);239 struct amd_pmc_dev *dev = dev_get_drvdata(d);
240+ int rc;
413 241 
414 if (!dev->major) {242 if (!dev->major) {
415- int rc = amd_pmc_get_smu_version(dev);243+ rc = amd_pmc_get_smu_version(dev);
416- 
417 if (rc)244 if (rc)
418 return rc;245 return rc;
419 }246 }
@@ -470,8 +297,8 @@ static int smu_fw_info_show(struct seq_file *s, void *unused)
470 297 
471 seq_puts(s, "\n=== Active time (in us) ===\n");298 seq_puts(s, "\n=== Active time (in us) ===\n");
472 for (idx = 0 ; idx < dev->num_ips ; idx++) {299 for (idx = 0 ; idx < dev->num_ips ; idx++) {
473- if (soc15_ip_blk[idx].bit_mask & dev->active_ips)300+ if (dev->ips_ptr[idx].bit_mask & dev->active_ips)
474- seq_printf(s, "%-8s : %lld\n", soc15_ip_blk[idx].name,301+ seq_printf(s, "%-8s : %lld\n", dev->ips_ptr[idx].name,
475 table.timecondition_notmet_lastcapture[idx]);302 table.timecondition_notmet_lastcapture[idx]);
476 }303 }
477 304 
@@ -538,6 +365,10 @@ static int amd_pmc_idlemask_read(struct amd_pmc_dev *pdev, struct device *dev,
538 case AMD_CPU_ID_PS:365 case AMD_CPU_ID_PS:
539 val = amd_pmc_reg_read(pdev, AMD_PMC_SCRATCH_REG_YC);366 val = amd_pmc_reg_read(pdev, AMD_PMC_SCRATCH_REG_YC);
540 break;367 break;
368+ case PCI_DEVICE_ID_AMD_1AH_M20H_ROOT:
369+ case PCI_DEVICE_ID_AMD_1AH_M60H_ROOT:
370+ val = amd_pmc_reg_read(pdev, AMD_PMC_SCRATCH_REG_1AH);
371+ break;
541 default:372 default:
542 return -EINVAL;373 return -EINVAL;
543 }374 }
@@ -562,18 +393,6 @@ static void amd_pmc_dbgfs_unregister(struct amd_pmc_dev *dev)
562 debugfs_remove_recursive(dev->dbgfs_dir);393 debugfs_remove_recursive(dev->dbgfs_dir);
563}394}
564 395 
565-static bool amd_pmc_is_stb_supported(struct amd_pmc_dev *dev)
566-{
567- switch (dev->cpu_id) {
568- case AMD_CPU_ID_YC:
569- case AMD_CPU_ID_CB:
570- case AMD_CPU_ID_PS:
571- return true;
572- default:
573- return false;
574- }
575-}
576- 
577static void amd_pmc_dbgfs_register(struct amd_pmc_dev *dev)396static void amd_pmc_dbgfs_register(struct amd_pmc_dev *dev)
578{397{
579 dev->dbgfs_dir = debugfs_create_dir("amd_pmc", NULL);398 dev->dbgfs_dir = debugfs_create_dir("amd_pmc", NULL);
@@ -583,14 +402,17 @@ static void amd_pmc_dbgfs_register(struct amd_pmc_dev *dev)
583 &s0ix_stats_fops);402 &s0ix_stats_fops);
584 debugfs_create_file("amd_pmc_idlemask", 0644, dev->dbgfs_dir, dev,403 debugfs_create_file("amd_pmc_idlemask", 0644, dev->dbgfs_dir, dev,
585 &amd_pmc_idlemask_fops);404 &amd_pmc_idlemask_fops);
586- /* Enable STB only when the module_param is set */405+}
587- if (enable_stb) {406+ 
588- if (amd_pmc_is_stb_supported(dev))407+static char *amd_pmc_get_msg_port(struct amd_pmc_dev *dev)
589- debugfs_create_file("stb_read", 0644, dev->dbgfs_dir, dev,408+{
590- &amd_pmc_stb_debugfs_fops_v2);409+ switch (dev->msg_port) {
591- else410+ case MSG_PORT_PMC:
592- debugfs_create_file("stb_read", 0644, dev->dbgfs_dir, dev,411+ return "PMC";
593- &amd_pmc_stb_debugfs_fops);412+ case MSG_PORT_S2D:
413+ return "S2D";
414+ default:
415+ return "Invalid message port";
594 }416 }
595}417}
596 418 
@@ -598,39 +420,39 @@ static void amd_pmc_dump_registers(struct amd_pmc_dev *dev)
598{420{
599 u32 value, message, argument, response;421 u32 value, message, argument, response;
600 422 
601- if (dev->msg_port) {423+ if (dev->msg_port == MSG_PORT_S2D) {
602- message = AMD_S2D_REGISTER_MESSAGE;424+ message = dev->stb_arg.msg;
603- argument = AMD_S2D_REGISTER_ARGUMENT;425+ argument = dev->stb_arg.arg;
604- response = AMD_S2D_REGISTER_RESPONSE;426+ response = dev->stb_arg.resp;
605 } else {427 } else {
606- message = AMD_PMC_REGISTER_MESSAGE;428+ message = dev->smu_msg;
607 argument = AMD_PMC_REGISTER_ARGUMENT;429 argument = AMD_PMC_REGISTER_ARGUMENT;
608 response = AMD_PMC_REGISTER_RESPONSE;430 response = AMD_PMC_REGISTER_RESPONSE;
609 }431 }
610 432 
611 value = amd_pmc_reg_read(dev, response);433 value = amd_pmc_reg_read(dev, response);
612- dev_dbg(dev->dev, "AMD_%s_REGISTER_RESPONSE:%x\n", dev->msg_port ? "S2D" : "PMC", value);434+ dev_dbg(dev->dev, "AMD_%s_REGISTER_RESPONSE:%x\n", amd_pmc_get_msg_port(dev), value);
613 435 
614 value = amd_pmc_reg_read(dev, argument);436 value = amd_pmc_reg_read(dev, argument);
615- dev_dbg(dev->dev, "AMD_%s_REGISTER_ARGUMENT:%x\n", dev->msg_port ? "S2D" : "PMC", value);437+ dev_dbg(dev->dev, "AMD_%s_REGISTER_ARGUMENT:%x\n", amd_pmc_get_msg_port(dev), value);
616 438 
617 value = amd_pmc_reg_read(dev, message);439 value = amd_pmc_reg_read(dev, message);
618- dev_dbg(dev->dev, "AMD_%s_REGISTER_MESSAGE:%x\n", dev->msg_port ? "S2D" : "PMC", value);440+ dev_dbg(dev->dev, "AMD_%s_REGISTER_MESSAGE:%x\n", amd_pmc_get_msg_port(dev), value);
619}441}
620 442 
621-static int amd_pmc_send_cmd(struct amd_pmc_dev *dev, u32 arg, u32 *data, u8 msg, bool ret)443+int amd_pmc_send_cmd(struct amd_pmc_dev *dev, u32 arg, u32 *data, u8 msg, bool ret)
622{444{
623 int rc;445 int rc;
624 u32 val, message, argument, response;446 u32 val, message, argument, response;
625 447 
626- mutex_lock(&dev->lock);448+ guard(mutex)(&dev->lock);
627 449 
628- if (dev->msg_port) {450+ if (dev->msg_port == MSG_PORT_S2D) {
629- message = AMD_S2D_REGISTER_MESSAGE;451+ message = dev->stb_arg.msg;
630- argument = AMD_S2D_REGISTER_ARGUMENT;452+ argument = dev->stb_arg.arg;
631- response = AMD_S2D_REGISTER_RESPONSE;453+ response = dev->stb_arg.resp;
632 } else {454 } else {
633- message = AMD_PMC_REGISTER_MESSAGE;455+ message = dev->smu_msg;
634 argument = AMD_PMC_REGISTER_ARGUMENT;456 argument = AMD_PMC_REGISTER_ARGUMENT;
635 response = AMD_PMC_REGISTER_RESPONSE;457 response = AMD_PMC_REGISTER_RESPONSE;
636 }458 }
@@ -641,7 +463,7 @@ static int amd_pmc_send_cmd(struct amd_pmc_dev *dev, u32 arg, u32 *data, u8 msg,
641 PMC_MSG_DELAY_MIN_US * RESPONSE_REGISTER_LOOP_MAX);463 PMC_MSG_DELAY_MIN_US * RESPONSE_REGISTER_LOOP_MAX);
642 if (rc) {464 if (rc) {
643 dev_err(dev->dev, "failed to talk to SMU\n");465 dev_err(dev->dev, "failed to talk to SMU\n");
644- goto out_unlock;466+ return rc;
645 }467 }
646 468 
647 /* Write zero to response register */469 /* Write zero to response register */
@@ -659,7 +481,7 @@ static int amd_pmc_send_cmd(struct amd_pmc_dev *dev, u32 arg, u32 *data, u8 msg,
659 PMC_MSG_DELAY_MIN_US * RESPONSE_REGISTER_LOOP_MAX);481 PMC_MSG_DELAY_MIN_US * RESPONSE_REGISTER_LOOP_MAX);
660 if (rc) {482 if (rc) {
661 dev_err(dev->dev, "SMU response timed out\n");483 dev_err(dev->dev, "SMU response timed out\n");
662- goto out_unlock;484+ return rc;
663 }485 }
664 486 
665 switch (val) {487 switch (val) {
@@ -673,21 +495,19 @@ static int amd_pmc_send_cmd(struct amd_pmc_dev *dev, u32 arg, u32 *data, u8 msg,
673 case AMD_PMC_RESULT_CMD_REJECT_BUSY:495 case AMD_PMC_RESULT_CMD_REJECT_BUSY:
674 dev_err(dev->dev, "SMU not ready. err: 0x%x\n", val);496 dev_err(dev->dev, "SMU not ready. err: 0x%x\n", val);
675 rc = -EBUSY;497 rc = -EBUSY;
676- goto out_unlock;498+ break;
677 case AMD_PMC_RESULT_CMD_UNKNOWN:499 case AMD_PMC_RESULT_CMD_UNKNOWN:
678 dev_err(dev->dev, "SMU cmd unknown. err: 0x%x\n", val);500 dev_err(dev->dev, "SMU cmd unknown. err: 0x%x\n", val);
679 rc = -EINVAL;501 rc = -EINVAL;
680- goto out_unlock;502+ break;
681 case AMD_PMC_RESULT_CMD_REJECT_PREREQ:503 case AMD_PMC_RESULT_CMD_REJECT_PREREQ:
682 case AMD_PMC_RESULT_FAILED:504 case AMD_PMC_RESULT_FAILED:
683 default:505 default:
684 dev_err(dev->dev, "SMU cmd failed. err: 0x%x\n", val);506 dev_err(dev->dev, "SMU cmd failed. err: 0x%x\n", val);
685 rc = -EIO;507 rc = -EIO;
686- goto out_unlock;508+ break;
687 }509 }
688 510 
689-out_unlock:
690- mutex_unlock(&dev->lock);
691 amd_pmc_dump_registers(dev);511 amd_pmc_dump_registers(dev);
692 return rc;512 return rc;
693}513}
@@ -698,9 +518,12 @@ static int amd_pmc_get_os_hint(struct amd_pmc_dev *dev)
698 case AMD_CPU_ID_PCO:518 case AMD_CPU_ID_PCO:
699 return MSG_OS_HINT_PCO;519 return MSG_OS_HINT_PCO;
700 case AMD_CPU_ID_RN:520 case AMD_CPU_ID_RN:
521+ case AMD_CPU_ID_VG:
701 case AMD_CPU_ID_YC:522 case AMD_CPU_ID_YC:
702 case AMD_CPU_ID_CB:523 case AMD_CPU_ID_CB:
703 case AMD_CPU_ID_PS:524 case AMD_CPU_ID_PS:
525+ case PCI_DEVICE_ID_AMD_1AH_M20H_ROOT:
526+ case PCI_DEVICE_ID_AMD_1AH_M60H_ROOT:
704 return MSG_OS_HINT_RN;527 return MSG_OS_HINT_RN;
705 }528 }
706 return -EINVAL;529 return -EINVAL;
@@ -709,19 +532,6 @@ static int amd_pmc_get_os_hint(struct amd_pmc_dev *dev)
709static int amd_pmc_wa_irq1(struct amd_pmc_dev *pdev)532static int amd_pmc_wa_irq1(struct amd_pmc_dev *pdev)
710{533{
711 struct device *d;534 struct device *d;
712- int rc;
713- 
714- /* cezanne platform firmware has a fix in 64.66.0 */
715- if (pdev->cpu_id == AMD_CPU_ID_CZN) {
716- if (!pdev->major) {
717- rc = amd_pmc_get_smu_version(pdev);
718- if (rc)
719- return rc;
720- }
721- 
722- if (pdev->major > 64 || (pdev->major == 64 && pdev->minor > 65))
723- return 0;
724- }
725 535 
726 d = bus_find_device_by_name(&serio_bus, NULL, "serio0");536 d = bus_find_device_by_name(&serio_bus, NULL, "serio0");
727 if (!d)537 if (!d)
@@ -814,7 +624,7 @@ static void amd_pmc_s2idle_prepare(void)
814 return;624 return;
815 }625 }
816 626 
817- rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_S2IDLE_PREPARE);627+ rc = amd_stb_write(pdev, AMD_PMC_STB_S2IDLE_PREPARE);
818 if (rc)628 if (rc)
819 dev_err(pdev->dev, "error writing to STB: %d\n", rc);629 dev_err(pdev->dev, "error writing to STB: %d\n", rc);
820}630}
@@ -832,7 +642,7 @@ static void amd_pmc_s2idle_check(void)
832 /* Dump the IdleMask before we add to the STB */642 /* Dump the IdleMask before we add to the STB */
833 amd_pmc_idlemask_read(pdev, pdev->dev, NULL);643 amd_pmc_idlemask_read(pdev, pdev->dev, NULL);
834 644 
835- rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_S2IDLE_CHECK);645+ rc = amd_stb_write(pdev, AMD_PMC_STB_S2IDLE_CHECK);
836 if (rc)646 if (rc)
837 dev_err(pdev->dev, "error writing to STB: %d\n", rc);647 dev_err(pdev->dev, "error writing to STB: %d\n", rc);
838}648}
@@ -859,7 +669,7 @@ static void amd_pmc_s2idle_restore(void)
859 /* Let SMU know that we are looking for stats */669 /* Let SMU know that we are looking for stats */
860 amd_pmc_dump_data(pdev);670 amd_pmc_dump_data(pdev);
861 671 
862- rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_S2IDLE_RESTORE);672+ rc = amd_stb_write(pdev, AMD_PMC_STB_S2IDLE_RESTORE);
863 if (rc)673 if (rc)
864 dev_err(pdev->dev, "error writing to STB: %d\n", rc);674 dev_err(pdev->dev, "error writing to STB: %d\n", rc);
865 675 
@@ -878,14 +688,14 @@ static struct acpi_s2idle_dev_ops amd_pmc_s2idle_dev_ops = {
878static int amd_pmc_suspend_handler(struct device *dev)688static int amd_pmc_suspend_handler(struct device *dev)
879{689{
880 struct amd_pmc_dev *pdev = dev_get_drvdata(dev);690 struct amd_pmc_dev *pdev = dev_get_drvdata(dev);
691+ int rc;
881 692 
882 /*693 /*
883 * Must be called only from the same set of dev_pm_ops handlers694 * Must be called only from the same set of dev_pm_ops handlers
884 * as i8042_pm_suspend() is called: currently just from .suspend.695 * as i8042_pm_suspend() is called: currently just from .suspend.
885 */696 */
886 if (pdev->disable_8042_wakeup && !disable_workarounds) {697 if (pdev->disable_8042_wakeup && !disable_workarounds) {
887- int rc = amd_pmc_wa_irq1(pdev);698+ rc = amd_pmc_wa_irq1(pdev);
888- 
889 if (rc) {699 if (rc) {
890 dev_err(pdev->dev, "failed to adjust keyboard wakeup: %d\n", rc);700 dev_err(pdev->dev, "failed to adjust keyboard wakeup: %d\n", rc);
891 return rc;701 return rc;
@@ -908,81 +718,13 @@ static const struct pci_device_id pmc_pci_ids[] = {
908 { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_PCO) },718 { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_PCO) },
909 { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_RV) },719 { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_RV) },
910 { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_SP) },720 { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_SP) },
721+ { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_SHP) },
722+ { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_VG) },
911 { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M20H_ROOT) },723 { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M20H_ROOT) },
724+ { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_1AH_M60H_ROOT) },
912 { }725 { }
913};726};
914 727 
915-static int amd_pmc_s2d_init(struct amd_pmc_dev *dev)
916-{
917- u32 phys_addr_low, phys_addr_hi;
918- u64 stb_phys_addr;
919- u32 size = 0;
920- int ret;
921- 
922- /* Spill to DRAM feature uses separate SMU message port */
923- dev->msg_port = 1;
924- 
925- /* Get num of IP blocks within the SoC */
926- amd_pmc_get_ip_info(dev);
927- 
928- amd_pmc_send_cmd(dev, S2D_TELEMETRY_SIZE, &size, dev->s2d_msg_id, true);
929- if (size != S2D_TELEMETRY_BYTES_MAX)
930- return -EIO;
931- 
932- /* Get DRAM size */
933- ret = amd_pmc_send_cmd(dev, S2D_DRAM_SIZE, &dev->dram_size, dev->s2d_msg_id, true);
934- if (ret || !dev->dram_size)
935- dev->dram_size = S2D_TELEMETRY_DRAMBYTES_MAX;
936- 
937- /* Get STB DRAM address */
938- amd_pmc_send_cmd(dev, S2D_PHYS_ADDR_LOW, &phys_addr_low, dev->s2d_msg_id, true);
939- amd_pmc_send_cmd(dev, S2D_PHYS_ADDR_HIGH, &phys_addr_hi, dev->s2d_msg_id, true);
940- 
941- if (!phys_addr_hi && !phys_addr_low) {
942- dev_err(dev->dev, "STB is not enabled on the system; disable enable_stb or contact system vendor\n");
943- return -EINVAL;
944- }
945- 
946- stb_phys_addr = ((u64)phys_addr_hi << 32 | phys_addr_low);
947- 
948- /* Clear msg_port for other SMU operation */
949- dev->msg_port = 0;
950- 
951- dev->stb_virt_addr = devm_ioremap(dev->dev, stb_phys_addr, dev->dram_size);
952- if (!dev->stb_virt_addr)
953- return -ENOMEM;
954- 
955- return 0;
956-}
957- 
958-static int amd_pmc_write_stb(struct amd_pmc_dev *dev, u32 data)
959-{
960- int err;
961- 
962- err = amd_smn_write(0, AMD_PMC_STB_PMI_0, data);
963- if (err) {
964- dev_err(dev->dev, "failed to write data in stb: 0x%X\n", AMD_PMC_STB_PMI_0);
965- return pcibios_err_to_errno(err);
966- }
967- 
968- return 0;
969-}
970- 
971-static int amd_pmc_read_stb(struct amd_pmc_dev *dev, u32 *buf)
972-{
973- int i, err;
974- 
975- for (i = 0; i < FIFO_SIZE; i++) {
976- err = amd_smn_read(0, AMD_PMC_STB_PMI_0, buf++);
977- if (err) {
978- dev_err(dev->dev, "error reading data from stb: 0x%X\n", AMD_PMC_STB_PMI_0);
979- return pcibios_err_to_errno(err);
980- }
981- }
982- 
983- return 0;
984-}
985- 
986static int amd_pmc_probe(struct platform_device *pdev)728static int amd_pmc_probe(struct platform_device *pdev)
987{729{
988 struct amd_pmc_dev *dev = &pmc;730 struct amd_pmc_dev *dev = &pmc;
@@ -993,7 +735,6 @@ static int amd_pmc_probe(struct platform_device *pdev)
993 u32 val;735 u32 val;
994 736 
995 dev->dev = &pdev->dev;737 dev->dev = &pdev->dev;
996- 
997 rdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0, 0));738 rdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0, 0));
998 if (!rdev || !pci_match_id(pmc_pci_ids, rdev)) {739 if (!rdev || !pci_match_id(pmc_pci_ids, rdev)) {
999 err = -ENODEV;740 err = -ENODEV;
@@ -1001,8 +742,7 @@ static int amd_pmc_probe(struct platform_device *pdev)
1001 }742 }
1002 743 
1003 dev->cpu_id = rdev->device;744 dev->cpu_id = rdev->device;
1004- 745+ if (dev->cpu_id == AMD_CPU_ID_SP || dev->cpu_id == AMD_CPU_ID_SHP) {
1005- if (dev->cpu_id == AMD_CPU_ID_SP) {
1006 dev_warn_once(dev->dev, "S0i3 is not supported on this hardware\n");746 dev_warn_once(dev->dev, "S0i3 is not supported on this hardware\n");
1007 err = -ENODEV;747 err = -ENODEV;
1008 goto err_pci_dev_put;748 goto err_pci_dev_put;
@@ -1017,7 +757,6 @@ static int amd_pmc_probe(struct platform_device *pdev)
1017 }757 }
1018 758 
1019 base_addr_lo = val & AMD_PMC_BASE_ADDR_HI_MASK;759 base_addr_lo = val & AMD_PMC_BASE_ADDR_HI_MASK;
1020- 
1021 err = amd_smn_read(0, AMD_PMC_BASE_ADDR_HI, &val);760 err = amd_smn_read(0, AMD_PMC_BASE_ADDR_HI, &val);
1022 if (err) {761 if (err) {
1023 dev_err(dev->dev, "error reading 0x%x\n", AMD_PMC_BASE_ADDR_HI);762 dev_err(dev->dev, "error reading 0x%x\n", AMD_PMC_BASE_ADDR_HI);
@@ -1035,13 +774,12 @@ static int amd_pmc_probe(struct platform_device *pdev)
1035 goto err_pci_dev_put;774 goto err_pci_dev_put;
1036 }775 }
1037 776 
1038- mutex_init(&dev->lock);777+ err = devm_mutex_init(dev->dev, &dev->lock);
778+ if (err)
779+ goto err_pci_dev_put;
1039 780 
1040- if (enable_stb && amd_pmc_is_stb_supported(dev)) {781+ /* Get num of IP blocks within the SoC */
1041- err = amd_pmc_s2d_init(dev);782+ amd_pmc_get_ip_info(dev);
1042- if (err)
1043- goto err_pci_dev_put;
1044- }
1045 783 
1046 platform_set_drvdata(pdev, dev);784 platform_set_drvdata(pdev, dev);
1047 if (IS_ENABLED(CONFIG_SUSPEND)) {785 if (IS_ENABLED(CONFIG_SUSPEND)) {
@@ -1053,6 +791,12 @@ static int amd_pmc_probe(struct platform_device *pdev)
1053 }791 }
1054 792 
1055 amd_pmc_dbgfs_register(dev);793 amd_pmc_dbgfs_register(dev);
794+ err = amd_stb_s2d_init(dev);
795+ if (err)
796+ goto err_pci_dev_put;
797+ 
798+ if (IS_ENABLED(CONFIG_AMD_MP2_STB))
799+ amd_mp2_stb_init(dev);
1056 pm_report_max_hw_sleep(U64_MAX);800 pm_report_max_hw_sleep(U64_MAX);
1057 return 0;801 return 0;
1058 802 
@@ -1069,7 +813,8 @@ static void amd_pmc_remove(struct platform_device *pdev)
1069 acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops);813 acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops);
1070 amd_pmc_dbgfs_unregister(dev);814 amd_pmc_dbgfs_unregister(dev);
1071 pci_dev_put(dev->rdev);815 pci_dev_put(dev->rdev);
1072- mutex_destroy(&dev->lock);816+ if (IS_ENABLED(CONFIG_AMD_MP2_STB))
817+ amd_mp2_stb_deinit(dev);
1073}818}
1074 819 
1075static const struct acpi_device_id amd_pmc_acpi_ids[] = {820static const struct acpi_device_id amd_pmc_acpi_ids[] = {
@@ -1079,6 +824,7 @@ static const struct acpi_device_id amd_pmc_acpi_ids[] = {
1079 {"AMDI0008", 0},824 {"AMDI0008", 0},
1080 {"AMDI0009", 0},825 {"AMDI0009", 0},
1081 {"AMDI000A", 0},826 {"AMDI000A", 0},
827+ {"AMDI000B", 0},
1082 {"AMD0004", 0},828 {"AMD0004", 0},
1083 {"AMD0005", 0},829 {"AMD0005", 0},
1084 { }830 { }
@@ -1093,7 +839,7 @@ static struct platform_driver amd_pmc_driver = {
1093 .pm = pm_sleep_ptr(&amd_pmc_pm),839 .pm = pm_sleep_ptr(&amd_pmc_pm),
1094 },840 },
1095 .probe = amd_pmc_probe,841 .probe = amd_pmc_probe,
1096- .remove_new = amd_pmc_remove,842+ .remove = amd_pmc_remove,
1097};843};
1098module_platform_driver(amd_pmc_driver);844module_platform_driver(amd_pmc_driver);
1099 845 
Mdrivers/platform/x86/amd/pmc/pmc.h+1-0
@@ -47,6 +47,7 @@ void amd_pmc_quirks_init(struct amd_pmc_dev *dev);
47#define AMD_CPU_ID_RN 0x163047#define AMD_CPU_ID_RN 0x1630
48#define AMD_CPU_ID_PCO AMD_CPU_ID_RV48#define AMD_CPU_ID_PCO AMD_CPU_ID_RV
49#define AMD_CPU_ID_CZN AMD_CPU_ID_RN49#define AMD_CPU_ID_CZN AMD_CPU_ID_RN
50+#define AMD_CPU_ID_VG 0x1645
50#define AMD_CPU_ID_YC 0x14B551#define AMD_CPU_ID_YC 0x14B5
51#define AMD_CPU_ID_CB 0x14D852#define AMD_CPU_ID_CB 0x14D8
52#define AMD_CPU_ID_PS 0x14E853#define AMD_CPU_ID_PS 0x14E8
Mfs/btrfs/space-info.c+472-125
Mfs/f2fs/f2fs.h+7-13
Mfs/f2fs/node.c+17-2
Mfs/gfs2/glock.c+368-399
Mfs/gfs2/inode.c+15-0
Mfs/gfs2/inode.h+1-0
Mfs/ntfs3/inode.c+170-194
Mfs/smb/client/dfs.c+317-203
Mmm/ksm.c+976-519