Star4
67
代码
代码
Issues2
Pull Requests3
流水线
项目讨论
Wiki
分析
Star4
67
已合并
cve漏洞修复 #289
chenyangw创建于 3月1日
cve漏洞修复 #289
已合并
chenyangw创建于 3月1日
从chenyang_w/kernel_linux_6.6_cve_remaincve_remain-fixmsg-72合入到openharmony/kernel_linux_6.6master
共 90 个文件变更+4853-3837
March/arm/mach-rockchip/platsmp.c+9-6
@@ -279,11 +279,6 @@ static void __init rockchip_smp_prepare_cpus(unsigned int max_cpus)
279 }279 }
280 280 
281 if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A9) {281 if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A9) {
282- if (rockchip_smp_prepare_sram(node)) {
283- of_node_put(node);
284- return;
285- }
286- 
287 /* enable the SCU power domain */282 /* enable the SCU power domain */
288 pmu_set_power_domain(PMU_PWRDN_SCU, true);283 pmu_set_power_domain(PMU_PWRDN_SCU, true);
289 284 
@@ -316,11 +311,19 @@ static void __init rockchip_smp_prepare_cpus(unsigned int max_cpus)
316 asm ("mrc p15, 1, %0, c9, c0, 2\n" : "=r" (l2ctlr));311 asm ("mrc p15, 1, %0, c9, c0, 2\n" : "=r" (l2ctlr));
317 ncores = ((l2ctlr >> 24) & 0x3) + 1;312 ncores = ((l2ctlr >> 24) & 0x3) + 1;
318 }313 }
319- of_node_put(node);
320 314 
321 /* Make sure that all cores except the first are really off */315 /* Make sure that all cores except the first are really off */
322 for (i = 1; i < ncores; i++)316 for (i = 1; i < ncores; i++)
323 pmu_set_power_domain(0 + i, false);317 pmu_set_power_domain(0 + i, false);
318+ 
319+ if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A9) {
320+ if (rockchip_smp_prepare_sram(node)) {
321+ of_node_put(node);
322+ return;
323+ }
324+ }
325+ 
326+ of_node_put(node);
324}327}
325 328 
326static void __init rk3036_smp_prepare_cpus(unsigned int max_cpus)329static void __init rk3036_smp_prepare_cpus(unsigned int max_cpus)
March/x86/events/amd/core.c+6-1
@@ -745,7 +745,12 @@ static void amd_pmu_enable_all(int added)
745 if (!test_bit(idx, cpuc->active_mask))745 if (!test_bit(idx, cpuc->active_mask))
746 continue;746 continue;
747 747 
748- amd_pmu_enable_event(cpuc->events[idx]);748+ /*
749+ * FIXME: cpuc->events[idx] can become NULL in a subtle race
750+ * condition with NMI->throttle->x86_pmu_stop().
751+ */
752+ if (cpuc->events[idx])
753+ amd_pmu_enable_event(cpuc->events[idx]);
749 }754 }
750}755}
751 756 
Mblock/blk-cgroup.c+8-15
@@ -848,14 +848,8 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
848 disk = ctx->bdev->bd_disk;848 disk = ctx->bdev->bd_disk;
849 q = disk->queue;849 q = disk->queue;
850 850 
851- /*851+ /* Prevent concurrent with blkcg_deactivate_policy() */
852- * blkcg_deactivate_policy() requires queue to be frozen, we can grab852+ mutex_lock(&q->blkcg_mutex);
853- * q_usage_counter to prevent concurrent with blkcg_deactivate_policy().
854- */
855- ret = blk_queue_enter(q, 0);
856- if (ret)
857- goto fail;
858- 
859 spin_lock_irq(&q->queue_lock);853 spin_lock_irq(&q->queue_lock);
860 854 
861 if (!blkcg_policy_enabled(q, pol)) {855 if (!blkcg_policy_enabled(q, pol)) {
@@ -885,16 +879,16 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
885 /* Drop locks to do new blkg allocation with GFP_KERNEL. */879 /* Drop locks to do new blkg allocation with GFP_KERNEL. */
886 spin_unlock_irq(&q->queue_lock);880 spin_unlock_irq(&q->queue_lock);
887 881 
888- new_blkg = blkg_alloc(pos, disk, GFP_KERNEL);882+ new_blkg = blkg_alloc(pos, disk, GFP_NOIO);
889 if (unlikely(!new_blkg)) {883 if (unlikely(!new_blkg)) {
890 ret = -ENOMEM;884 ret = -ENOMEM;
891- goto fail_exit_queue;885+ goto fail_exit;
892 }886 }
893 887 
894 if (radix_tree_preload(GFP_KERNEL)) {888 if (radix_tree_preload(GFP_KERNEL)) {
895 blkg_free(new_blkg);889 blkg_free(new_blkg);
896 ret = -ENOMEM;890 ret = -ENOMEM;
897- goto fail_exit_queue;891+ goto fail_exit;
898 }892 }
899 893 
900 spin_lock_irq(&q->queue_lock);894 spin_lock_irq(&q->queue_lock);
@@ -922,7 +916,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
922 goto success;916 goto success;
923 }917 }
924success:918success:
925- blk_queue_exit(q);919+ mutex_unlock(&q->blkcg_mutex);
926 ctx->blkg = blkg;920 ctx->blkg = blkg;
927 return 0;921 return 0;
928 922 
@@ -930,9 +924,8 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
930 radix_tree_preload_end();924 radix_tree_preload_end();
931fail_unlock:925fail_unlock:
932 spin_unlock_irq(&q->queue_lock);926 spin_unlock_irq(&q->queue_lock);
933-fail_exit_queue:927+fail_exit:
934- blk_queue_exit(q);928+ mutex_unlock(&q->blkcg_mutex);
935-fail:
936 /*929 /*
937 * If queue was bypassing, we should retry. Do so after a930 * If queue was bypassing, we should retry. Do so after a
938 * short msleep(). It isn't strictly necessary but queue931 * short msleep(). It isn't strictly necessary but queue
Mblock/blk-mq.c+8-9
@@ -279,12 +279,12 @@ void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set)
279{279{
280 struct request_queue *q;280 struct request_queue *q;
281 281 
282- mutex_lock(&set->tag_list_lock);282+ rcu_read_lock();
283- list_for_each_entry(q, &set->tag_list, tag_set_list) {283+ list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) {
284 if (!blk_queue_skip_tagset_quiesce(q))284 if (!blk_queue_skip_tagset_quiesce(q))
285 blk_mq_quiesce_queue_nowait(q);285 blk_mq_quiesce_queue_nowait(q);
286 }286 }
287- mutex_unlock(&set->tag_list_lock);287+ rcu_read_unlock();
288 288 
289 blk_mq_wait_quiesce_done(set);289 blk_mq_wait_quiesce_done(set);
290}290}
@@ -294,12 +294,12 @@ void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set)
294{294{
295 struct request_queue *q;295 struct request_queue *q;
296 296 
297- mutex_lock(&set->tag_list_lock);297+ rcu_read_lock();
298- list_for_each_entry(q, &set->tag_list, tag_set_list) {298+ list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) {
299 if (!blk_queue_skip_tagset_quiesce(q))299 if (!blk_queue_skip_tagset_quiesce(q))
300 blk_mq_unquiesce_queue(q);300 blk_mq_unquiesce_queue(q);
301 }301 }
302- mutex_unlock(&set->tag_list_lock);302+ rcu_read_unlock();
303}303}
304EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset);304EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset);
305 305 
@@ -4103,7 +4103,7 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q)
4103 struct blk_mq_tag_set *set = q->tag_set;4103 struct blk_mq_tag_set *set = q->tag_set;
4104 4104 
4105 mutex_lock(&set->tag_list_lock);4105 mutex_lock(&set->tag_list_lock);
4106- list_del(&q->tag_set_list);4106+ list_del_rcu(&q->tag_set_list);
4107 if (list_is_singular(&set->tag_list)) {4107 if (list_is_singular(&set->tag_list)) {
4108 /* just transitioned to unshared */4108 /* just transitioned to unshared */
4109 set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;4109 set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
@@ -4111,7 +4111,6 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q)
4111 blk_mq_update_tag_set_shared(set, false);4111 blk_mq_update_tag_set_shared(set, false);
4112 }4112 }
4113 mutex_unlock(&set->tag_list_lock);4113 mutex_unlock(&set->tag_list_lock);
4114- INIT_LIST_HEAD(&q->tag_set_list);
4115}4114}
4116 4115 
4117static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,4116static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
@@ -4130,7 +4129,7 @@ static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
4130 }4129 }
4131 if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)4130 if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
4132 queue_set_hctx_shared(q, true);4131 queue_set_hctx_shared(q, true);
4133- list_add_tail(&q->tag_set_list, &set->tag_list);4132+ list_add_tail_rcu(&q->tag_set_list, &set->tag_list);
4134 4133 
4135 mutex_unlock(&set->tag_list_lock);4134 mutex_unlock(&set->tag_list_lock);
4136}4135}
Mblock/blk-settings.c+1-1
@@ -634,7 +634,7 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
634 }634 }
635 635 
636 /* chunk_sectors a multiple of the physical block size? */636 /* chunk_sectors a multiple of the physical block size? */
637- if ((t->chunk_sectors << 9) & (t->physical_block_size - 1)) {637+ if (t->chunk_sectors % (t->physical_block_size >> SECTOR_SHIFT)) {
638 t->chunk_sectors = 0;638 t->chunk_sectors = 0;
639 t->misaligned = 1;639 t->misaligned = 1;
640 ret = -1;640 ret = -1;
Mcrypto/asymmetric_keys/asymmetric_type.c+9-3
@@ -11,6 +11,7 @@
11#include <crypto/public_key.h>11#include <crypto/public_key.h>
12#include <linux/seq_file.h>12#include <linux/seq_file.h>
13#include <linux/module.h>13#include <linux/module.h>
14+#include <linux/overflow.h>
14#include <linux/slab.h>15#include <linux/slab.h>
15#include <linux/ctype.h>16#include <linux/ctype.h>
16#include <keys/system_keyring.h>17#include <keys/system_keyring.h>
@@ -151,12 +152,17 @@ struct asymmetric_key_id *asymmetric_key_generate_id(const void *val_1,
151 size_t len_2)152 size_t len_2)
152{153{
153 struct asymmetric_key_id *kid;154 struct asymmetric_key_id *kid;
155+ size_t kid_sz;
156+ size_t len;
154 157 
155- kid = kmalloc(sizeof(struct asymmetric_key_id) + len_1 + len_2,158+ if (check_add_overflow(len_1, len_2, &len))
156- GFP_KERNEL);159+ return ERR_PTR(-EOVERFLOW);
160+ if (check_add_overflow(sizeof(struct asymmetric_key_id), len, &kid_sz))
161+ return ERR_PTR(-EOVERFLOW);
162+ kid = kmalloc(kid_sz, GFP_KERNEL);
157 if (!kid)163 if (!kid)
158 return ERR_PTR(-ENOMEM);164 return ERR_PTR(-ENOMEM);
159- kid->len = len_1 + len_2;165+ kid->len = len;
160 memcpy(kid->data, val_1, len_1);166 memcpy(kid->data, val_1, len_1);
161 memcpy(kid->data + len_1, val_2, len_2);167 memcpy(kid->data + len_1, val_2, len_2);
162 return kid;168 return kid;
Mdrivers/block/nbd.c+1-1
@@ -912,9 +912,9 @@ static void recv_work(struct work_struct *work)
912 nbd_mark_nsock_dead(nbd, nsock, 1);912 nbd_mark_nsock_dead(nbd, nsock, 1);
913 mutex_unlock(&nsock->tx_lock);913 mutex_unlock(&nsock->tx_lock);
914 914 
915- nbd_config_put(nbd);
916 atomic_dec(&config->recv_threads);915 atomic_dec(&config->recv_threads);
917 wake_up(&config->recv_wq);916 wake_up(&config->recv_wq);
917+ nbd_config_put(nbd);
918 kfree(args);918 kfree(args);
919}919}
920 920 
Mdrivers/crypto/starfive/jh7110-hash.c+5-1
@@ -358,6 +358,7 @@ static int starfive_hash_digest(struct ahash_request *req)
358 struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);358 struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
359 struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);359 struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
360 struct starfive_cryp_dev *cryp = ctx->cryp;360 struct starfive_cryp_dev *cryp = ctx->cryp;
361+ int sg_len;
361 362 
362 memset(rctx, 0, sizeof(struct starfive_cryp_request_ctx));363 memset(rctx, 0, sizeof(struct starfive_cryp_request_ctx));
363 364 
@@ -366,7 +367,10 @@ static int starfive_hash_digest(struct ahash_request *req)
366 rctx->in_sg = req->src;367 rctx->in_sg = req->src;
367 rctx->blksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));368 rctx->blksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
368 rctx->digsize = crypto_ahash_digestsize(tfm);369 rctx->digsize = crypto_ahash_digestsize(tfm);
369- rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total);370+ sg_len = sg_nents_for_len(rctx->in_sg, rctx->total);
371+ if (sg_len < 0)
372+ return sg_len;
373+ rctx->in_sg_len = sg_len;
370 ctx->rctx = rctx;374 ctx->rctx = rctx;
371 375 
372 if (starfive_hash_check_aligned(rctx->in_sg, rctx->total, rctx->blksize))376 if (starfive_hash_check_aligned(rctx->in_sg, rctx->total, rctx->blksize))
Mdrivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c+601-286
@@ -24,16 +24,16 @@
24#include "soc15.h"24#include "soc15.h"
25 25 
26#include "soc15_common.h"26#include "soc15_common.h"
27+#include "amdgpu_reg_state.h"
27#include "amdgpu_xcp.h"28#include "amdgpu_xcp.h"
28#include "gfx_v9_4_3.h"29#include "gfx_v9_4_3.h"
29#include "gfxhub_v1_2.h"30#include "gfxhub_v1_2.h"
30#include "sdma_v4_4_2.h"31#include "sdma_v4_4_2.h"
32+#include "amdgpu_ip.h"
31 33 
32#define XCP_INST_MASK(num_inst, xcp_id) \34#define XCP_INST_MASK(num_inst, xcp_id) \
33 (num_inst ? GENMASK(num_inst - 1, 0) << (xcp_id * num_inst) : 0)35 (num_inst ? GENMASK(num_inst - 1, 0) << (xcp_id * num_inst) : 0)
34 36 
35-#define AMDGPU_XCP_OPS_KFD (1 << 0)
36- 
37void aqua_vanjaram_doorbell_index_init(struct amdgpu_device *adev)37void aqua_vanjaram_doorbell_index_init(struct amdgpu_device *adev)
38{38{
39 int i;39 int i;
@@ -61,229 +61,6 @@ void aqua_vanjaram_doorbell_index_init(struct amdgpu_device *adev)
61 adev->doorbell_index.max_assignment = AMDGPU_DOORBELL_LAYOUT1_MAX_ASSIGNMENT << 1;61 adev->doorbell_index.max_assignment = AMDGPU_DOORBELL_LAYOUT1_MAX_ASSIGNMENT << 1;
62}62}
63 63 
64-static bool aqua_vanjaram_xcp_vcn_shared(struct amdgpu_device *adev)
65-{
66- return (adev->xcp_mgr->num_xcps > adev->vcn.num_vcn_inst);
67-}
68- 
69-static void aqua_vanjaram_set_xcp_id(struct amdgpu_device *adev,
70- uint32_t inst_idx, struct amdgpu_ring *ring)
71-{
72- int xcp_id;
73- enum AMDGPU_XCP_IP_BLOCK ip_blk;
74- uint32_t inst_mask;
75- 
76- ring->xcp_id = AMDGPU_XCP_NO_PARTITION;
77- if (adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE)
78- return;
79- 
80- inst_mask = 1 << inst_idx;
81- 
82- switch (ring->funcs->type) {
83- case AMDGPU_HW_IP_GFX:
84- case AMDGPU_RING_TYPE_COMPUTE:
85- case AMDGPU_RING_TYPE_KIQ:
86- ip_blk = AMDGPU_XCP_GFX;
87- break;
88- case AMDGPU_RING_TYPE_SDMA:
89- ip_blk = AMDGPU_XCP_SDMA;
90- break;
91- case AMDGPU_RING_TYPE_VCN_ENC:
92- case AMDGPU_RING_TYPE_VCN_JPEG:
93- ip_blk = AMDGPU_XCP_VCN;
94- if (aqua_vanjaram_xcp_vcn_shared(adev))
95- inst_mask = 1 << (inst_idx * 2);
96- break;
97- default:
98- DRM_ERROR("Not support ring type %d!", ring->funcs->type);
99- return;
100- }
101- 
102- for (xcp_id = 0; xcp_id < adev->xcp_mgr->num_xcps; xcp_id++) {
103- if (adev->xcp_mgr->xcp[xcp_id].ip[ip_blk].inst_mask & inst_mask) {
104- ring->xcp_id = xcp_id;
105- break;
106- }
107- }
108-}
109- 
110-static void aqua_vanjaram_xcp_gpu_sched_update(
111- struct amdgpu_device *adev,
112- struct amdgpu_ring *ring,
113- unsigned int sel_xcp_id)
114-{
115- unsigned int *num_gpu_sched;
116- 
117- num_gpu_sched = &adev->xcp_mgr->xcp[sel_xcp_id]
118- .gpu_sched[ring->funcs->type][ring->hw_prio].num_scheds;
119- adev->xcp_mgr->xcp[sel_xcp_id].gpu_sched[ring->funcs->type][ring->hw_prio]
120- .sched[(*num_gpu_sched)++] = &ring->sched;
121- DRM_DEBUG("%s :[%d] gpu_sched[%d][%d] = %d", ring->name,
122- sel_xcp_id, ring->funcs->type,
123- ring->hw_prio, *num_gpu_sched);
124-}
125- 
126-static int aqua_vanjaram_xcp_sched_list_update(
127- struct amdgpu_device *adev)
128-{
129- struct amdgpu_ring *ring;
130- int i;
131- 
132- for (i = 0; i < MAX_XCP; i++) {
133- atomic_set(&adev->xcp_mgr->xcp[i].ref_cnt, 0);
134- memset(adev->xcp_mgr->xcp[i].gpu_sched, 0, sizeof(adev->xcp_mgr->xcp->gpu_sched));
135- }
136- 
137- if (adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE)
138- return 0;
139- 
140- for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
141- ring = adev->rings[i];
142- if (!ring || !ring->sched.ready || ring->no_scheduler)
143- continue;
144- 
145- aqua_vanjaram_xcp_gpu_sched_update(adev, ring, ring->xcp_id);
146- 
147- /* VCN may be shared by two partitions under CPX MODE in certain
148- * configs.
149- */
150- if ((ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC ||
151- ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG) &&
152- aqua_vanjaram_xcp_vcn_shared(adev))
153- aqua_vanjaram_xcp_gpu_sched_update(adev, ring, ring->xcp_id + 1);
154- }
155- 
156- return 0;
157-}
158- 
159-static int aqua_vanjaram_update_partition_sched_list(struct amdgpu_device *adev)
160-{
161- int i;
162- 
163- for (i = 0; i < adev->num_rings; i++) {
164- struct amdgpu_ring *ring = adev->rings[i];
165- 
166- if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE ||
167- ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
168- aqua_vanjaram_set_xcp_id(adev, ring->xcc_id, ring);
169- else
170- aqua_vanjaram_set_xcp_id(adev, ring->me, ring);
171- }
172- 
173- return aqua_vanjaram_xcp_sched_list_update(adev);
174-}
175- 
176-static int aqua_vanjaram_select_scheds(
177- struct amdgpu_device *adev,
178- u32 hw_ip,
179- u32 hw_prio,
180- struct amdgpu_fpriv *fpriv,
181- unsigned int *num_scheds,
182- struct drm_gpu_scheduler ***scheds)
183-{
184- u32 sel_xcp_id;
185- int i;
186- 
187- if (fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION) {
188- u32 least_ref_cnt = ~0;
189- 
190- fpriv->xcp_id = 0;
191- for (i = 0; i < adev->xcp_mgr->num_xcps; i++) {
192- u32 total_ref_cnt;
193- 
194- total_ref_cnt = atomic_read(&adev->xcp_mgr->xcp[i].ref_cnt);
195- if (total_ref_cnt < least_ref_cnt) {
196- fpriv->xcp_id = i;
197- least_ref_cnt = total_ref_cnt;
198- }
199- }
200- }
201- sel_xcp_id = fpriv->xcp_id;
202- 
203- if (adev->xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds) {
204- *num_scheds = adev->xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds;
205- *scheds = adev->xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].sched;
206- atomic_inc(&adev->xcp_mgr->xcp[sel_xcp_id].ref_cnt);
207- DRM_DEBUG("Selected partition #%d", sel_xcp_id);
208- } else {
209- DRM_ERROR("Failed to schedule partition #%d.", sel_xcp_id);
210- return -ENOENT;
211- }
212- 
213- return 0;
214-}
215- 
216-static int8_t aqua_vanjaram_logical_to_dev_inst(struct amdgpu_device *adev,
217- enum amd_hw_ip_block_type block,
218- int8_t inst)
219-{
220- int8_t dev_inst;
221- 
222- switch (block) {
223- case GC_HWIP:
224- case SDMA0_HWIP:
225- /* Both JPEG and VCN as JPEG is only alias of VCN */
226- case VCN_HWIP:
227- dev_inst = adev->ip_map.dev_inst[block][inst];
228- break;
229- default:
230- /* For rest of the IPs, no look up required.
231- * Assume 'logical instance == physical instance' for all configs. */
232- dev_inst = inst;
233- break;
234- }
235- 
236- return dev_inst;
237-}
238- 
239-static uint32_t aqua_vanjaram_logical_to_dev_mask(struct amdgpu_device *adev,
240- enum amd_hw_ip_block_type block,
241- uint32_t mask)
242-{
243- uint32_t dev_mask = 0;
244- int8_t log_inst, dev_inst;
245- 
246- while (mask) {
247- log_inst = ffs(mask) - 1;
248- dev_inst = aqua_vanjaram_logical_to_dev_inst(adev, block, log_inst);
249- dev_mask |= (1 << dev_inst);
250- mask &= ~(1 << log_inst);
251- }
252- 
253- return dev_mask;
254-}
255- 
256-static void aqua_vanjaram_populate_ip_map(struct amdgpu_device *adev,
257- enum amd_hw_ip_block_type ip_block,
258- uint32_t inst_mask)
259-{
260- int l = 0, i;
261- 
262- while (inst_mask) {
263- i = ffs(inst_mask) - 1;
264- adev->ip_map.dev_inst[ip_block][l++] = i;
265- inst_mask &= ~(1 << i);
266- }
267- for (; l < HWIP_MAX_INSTANCE; l++)
268- adev->ip_map.dev_inst[ip_block][l] = -1;
269-}
270- 
271-void aqua_vanjaram_ip_map_init(struct amdgpu_device *adev)
272-{
273- u32 ip_map[][2] = {
274- { GC_HWIP, adev->gfx.xcc_mask },
275- { SDMA0_HWIP, adev->sdma.sdma_mask },
276- { VCN_HWIP, adev->vcn.inst_mask },
277- };
278- int i;
279- 
280- for (i = 0; i < ARRAY_SIZE(ip_map); ++i)
281- aqua_vanjaram_populate_ip_map(adev, ip_map[i][0], ip_map[i][1]);
282- 
283- adev->ip_map.logical_to_dev_inst = aqua_vanjaram_logical_to_dev_inst;
284- adev->ip_map.logical_to_dev_mask = aqua_vanjaram_logical_to_dev_mask;
285-}
286- 
287/* Fixed pattern for smn addressing on different AIDs:64/* Fixed pattern for smn addressing on different AIDs:
288 * bit[34]: indicate cross AID access65 * bit[34]: indicate cross AID access
289 * bit[33:32]: indicate target AID id66 * bit[33:32]: indicate target AID id
@@ -303,13 +80,59 @@ u64 aqua_vanjaram_encode_ext_smn_addressing(int ext_id)
303 return ext_offset;80 return ext_offset;
304}81}
305 82 
83+static enum amdgpu_gfx_partition
84+__aqua_vanjaram_calc_xcp_mode(struct amdgpu_xcp_mgr *xcp_mgr)
85+{
86+ struct amdgpu_device *adev = xcp_mgr->adev;
87+ int num_xcc, num_xcc_per_xcp = 0, mode = 0;
88+ 
89+ num_xcc = NUM_XCC(xcp_mgr->adev->gfx.xcc_mask);
90+ if (adev->gfx.funcs->get_xccs_per_xcp)
91+ num_xcc_per_xcp = adev->gfx.funcs->get_xccs_per_xcp(adev);
92+ if ((num_xcc_per_xcp) && (num_xcc % num_xcc_per_xcp == 0))
93+ mode = num_xcc / num_xcc_per_xcp;
94+ 
95+ if (num_xcc_per_xcp == 1)
96+ return AMDGPU_CPX_PARTITION_MODE;
97+ 
98+ switch (mode) {
99+ case 1:
100+ return AMDGPU_SPX_PARTITION_MODE;
101+ case 2:
102+ return AMDGPU_DPX_PARTITION_MODE;
103+ case 3:
104+ return AMDGPU_TPX_PARTITION_MODE;
105+ case 4:
106+ return AMDGPU_QPX_PARTITION_MODE;
107+ default:
108+ return AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE;
109+ }
110+ 
111+ return AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE;
112+}
113+ 
306static int aqua_vanjaram_query_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr)114static int aqua_vanjaram_query_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr)
307{115{
308- enum amdgpu_gfx_partition mode = AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE;116+ enum amdgpu_gfx_partition derv_mode,
117+ mode = AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE;
309 struct amdgpu_device *adev = xcp_mgr->adev;118 struct amdgpu_device *adev = xcp_mgr->adev;
310 119 
311- if (adev->nbio.funcs->get_compute_partition_mode)120+ derv_mode = __aqua_vanjaram_calc_xcp_mode(xcp_mgr);
121+ 
122+ if (amdgpu_sriov_vf(adev))
123+ return derv_mode;
124+ 
125+ if (adev->nbio.funcs->get_compute_partition_mode) {
312 mode = adev->nbio.funcs->get_compute_partition_mode(adev);126 mode = adev->nbio.funcs->get_compute_partition_mode(adev);
127+ if (mode != derv_mode) {
128+ dev_warn(
129+ adev->dev,
130+ "Mismatch in compute partition mode - reported : %d derived : %d",
131+ mode, derv_mode);
132+ if (derv_mode == AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE)
133+ amdgpu_device_bus_status_check(adev);
134+ }
135+ }
313 136 
314 return mode;137 return mode;
315}138}
@@ -346,38 +169,31 @@ static int __aqua_vanjaram_get_xcp_ip_info(struct amdgpu_xcp_mgr *xcp_mgr, int x
346 struct amdgpu_xcp_ip *ip)169 struct amdgpu_xcp_ip *ip)
347{170{
348 struct amdgpu_device *adev = xcp_mgr->adev;171 struct amdgpu_device *adev = xcp_mgr->adev;
172+ int num_sdma, num_vcn, num_shared_vcn, num_xcp;
349 int num_xcc_xcp, num_sdma_xcp, num_vcn_xcp;173 int num_xcc_xcp, num_sdma_xcp, num_vcn_xcp;
350- int num_sdma, num_vcn;
351 174 
352 num_sdma = adev->sdma.num_instances;175 num_sdma = adev->sdma.num_instances;
353 num_vcn = adev->vcn.num_vcn_inst;176 num_vcn = adev->vcn.num_vcn_inst;
177+ num_shared_vcn = 1;
178+ 
179+ num_xcc_xcp = adev->gfx.num_xcc_per_xcp;
180+ num_xcp = NUM_XCC(adev->gfx.xcc_mask) / num_xcc_xcp;
354 181 
355 switch (xcp_mgr->mode) {182 switch (xcp_mgr->mode) {
356 case AMDGPU_SPX_PARTITION_MODE:183 case AMDGPU_SPX_PARTITION_MODE:
357- num_sdma_xcp = num_sdma;
358- num_vcn_xcp = num_vcn;
359- break;
360 case AMDGPU_DPX_PARTITION_MODE:184 case AMDGPU_DPX_PARTITION_MODE:
361- num_sdma_xcp = num_sdma / 2;
362- num_vcn_xcp = num_vcn / 2;
363- break;
364 case AMDGPU_TPX_PARTITION_MODE:185 case AMDGPU_TPX_PARTITION_MODE:
365- num_sdma_xcp = num_sdma / 3;
366- num_vcn_xcp = num_vcn / 3;
367- break;
368 case AMDGPU_QPX_PARTITION_MODE:186 case AMDGPU_QPX_PARTITION_MODE:
369- num_sdma_xcp = num_sdma / 4;
370- num_vcn_xcp = num_vcn / 4;
371- break;
372 case AMDGPU_CPX_PARTITION_MODE:187 case AMDGPU_CPX_PARTITION_MODE:
373- num_sdma_xcp = 2;188+ num_sdma_xcp = DIV_ROUND_UP(num_sdma, num_xcp);
374- num_vcn_xcp = num_vcn ? 1 : 0;189+ num_vcn_xcp = DIV_ROUND_UP(num_vcn, num_xcp);
375 break;190 break;
376 default:191 default:
377 return -EINVAL;192 return -EINVAL;
378 }193 }
379 194 
380- num_xcc_xcp = adev->gfx.num_xcc_per_xcp;195+ if (num_vcn && num_xcp > num_vcn)
196+ num_shared_vcn = num_xcp / num_vcn;
381 197 
382 switch (ip_id) {198 switch (ip_id) {
383 case AMDGPU_XCP_GFXHUB:199 case AMDGPU_XCP_GFXHUB:
@@ -393,7 +209,8 @@ static int __aqua_vanjaram_get_xcp_ip_info(struct amdgpu_xcp_mgr *xcp_mgr, int x
393 ip->ip_funcs = &sdma_v4_4_2_xcp_funcs;209 ip->ip_funcs = &sdma_v4_4_2_xcp_funcs;
394 break;210 break;
395 case AMDGPU_XCP_VCN:211 case AMDGPU_XCP_VCN:
396- ip->inst_mask = XCP_INST_MASK(num_vcn_xcp, xcp_id);212+ ip->inst_mask =
213+ XCP_INST_MASK(num_vcn_xcp, xcp_id / num_shared_vcn);
397 /* TODO : Assign IP funcs */214 /* TODO : Assign IP funcs */
398 break;215 break;
399 default:216 default:
@@ -405,6 +222,94 @@ static int __aqua_vanjaram_get_xcp_ip_info(struct amdgpu_xcp_mgr *xcp_mgr, int x
405 return 0;222 return 0;
406}223}
407 224 
225+static int __aqua_vanjaram_get_px_mode_info(struct amdgpu_xcp_mgr *xcp_mgr,
226+ int px_mode, int *num_xcp,
227+ uint16_t *nps_modes)
228+{
229+ struct amdgpu_device *adev = xcp_mgr->adev;
230+ uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
231+ 
232+ if (!num_xcp || !nps_modes || !(xcp_mgr->supp_xcp_modes & BIT(px_mode)))
233+ return -EINVAL;
234+ 
235+ switch (px_mode) {
236+ case AMDGPU_SPX_PARTITION_MODE:
237+ *num_xcp = 1;
238+ *nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE);
239+ break;
240+ case AMDGPU_DPX_PARTITION_MODE:
241+ *num_xcp = 2;
242+ *nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE) |
243+ BIT(AMDGPU_NPS2_PARTITION_MODE);
244+ break;
245+ case AMDGPU_TPX_PARTITION_MODE:
246+ *num_xcp = 3;
247+ *nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE) |
248+ BIT(AMDGPU_NPS4_PARTITION_MODE);
249+ break;
250+ case AMDGPU_QPX_PARTITION_MODE:
251+ *num_xcp = 4;
252+ *nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE) |
253+ BIT(AMDGPU_NPS4_PARTITION_MODE);
254+ if (gc_ver == IP_VERSION(9, 5, 0))
255+ *nps_modes |= BIT(AMDGPU_NPS2_PARTITION_MODE);
256+ break;
257+ case AMDGPU_CPX_PARTITION_MODE:
258+ *num_xcp = NUM_XCC(adev->gfx.xcc_mask);
259+ *nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE) |
260+ BIT(AMDGPU_NPS4_PARTITION_MODE);
261+ if (gc_ver == IP_VERSION(9, 5, 0))
262+ *nps_modes |= BIT(AMDGPU_NPS2_PARTITION_MODE);
263+ break;
264+ default:
265+ return -EINVAL;
266+ }
267+ 
268+ return 0;
269+}
270+ 
271+static int aqua_vanjaram_get_xcp_res_info(struct amdgpu_xcp_mgr *xcp_mgr,
272+ int mode,
273+ struct amdgpu_xcp_cfg *xcp_cfg)
274+{
275+ struct amdgpu_device *adev = xcp_mgr->adev;
276+ int max_res[AMDGPU_XCP_RES_MAX] = {};
277+ bool res_lt_xcp;
278+ int num_xcp, i, r;
279+ u16 nps_modes;
280+ 
281+ if (!(xcp_mgr->supp_xcp_modes & BIT(mode)))
282+ return -EINVAL;
283+ 
284+ max_res[AMDGPU_XCP_RES_XCC] = NUM_XCC(adev->gfx.xcc_mask);
285+ max_res[AMDGPU_XCP_RES_DMA] = adev->sdma.num_instances;
286+ max_res[AMDGPU_XCP_RES_DEC] = adev->vcn.num_vcn_inst;
287+ max_res[AMDGPU_XCP_RES_JPEG] = adev->jpeg.num_jpeg_inst;
288+ 
289+ r = __aqua_vanjaram_get_px_mode_info(xcp_mgr, mode, &num_xcp, &nps_modes);
290+ if (r)
291+ return r;
292+ 
293+ xcp_cfg->compatible_nps_modes =
294+ (adev->gmc.supported_nps_modes & nps_modes);
295+ xcp_cfg->num_res = ARRAY_SIZE(max_res);
296+ 
297+ for (i = 0; i < xcp_cfg->num_res; i++) {
298+ res_lt_xcp = max_res[i] < num_xcp;
299+ xcp_cfg->xcp_res[i].id = i;
300+ xcp_cfg->xcp_res[i].num_inst =
301+ res_lt_xcp ? 1 : max_res[i] / num_xcp;
302+ xcp_cfg->xcp_res[i].num_inst =
303+ i == AMDGPU_XCP_RES_JPEG ?
304+ xcp_cfg->xcp_res[i].num_inst *
305+ adev->jpeg.num_jpeg_rings : xcp_cfg->xcp_res[i].num_inst;
306+ xcp_cfg->xcp_res[i].num_shared =
307+ res_lt_xcp ? num_xcp / max_res[i] : 1;
308+ }
309+ 
310+ return 0;
311+}
312+ 
408static enum amdgpu_gfx_partition313static enum amdgpu_gfx_partition
409__aqua_vanjaram_get_auto_mode(struct amdgpu_xcp_mgr *xcp_mgr)314__aqua_vanjaram_get_auto_mode(struct amdgpu_xcp_mgr *xcp_mgr)
410{315{
@@ -421,7 +326,7 @@ __aqua_vanjaram_get_auto_mode(struct amdgpu_xcp_mgr *xcp_mgr)
421 326 
422 if (adev->gmc.num_mem_partitions == num_xcc / 2)327 if (adev->gmc.num_mem_partitions == num_xcc / 2)
423 return (adev->flags & AMD_IS_APU) ? AMDGPU_TPX_PARTITION_MODE :328 return (adev->flags & AMD_IS_APU) ? AMDGPU_TPX_PARTITION_MODE :
424- AMDGPU_QPX_PARTITION_MODE;329+ AMDGPU_CPX_PARTITION_MODE;
425 330 
426 if (adev->gmc.num_mem_partitions == 2 && !(adev->flags & AMD_IS_APU))331 if (adev->gmc.num_mem_partitions == 2 && !(adev->flags & AMD_IS_APU))
427 return AMDGPU_DPX_PARTITION_MODE;332 return AMDGPU_DPX_PARTITION_MODE;
@@ -433,27 +338,31 @@ static bool __aqua_vanjaram_is_valid_mode(struct amdgpu_xcp_mgr *xcp_mgr,
433 enum amdgpu_gfx_partition mode)338 enum amdgpu_gfx_partition mode)
434{339{
435 struct amdgpu_device *adev = xcp_mgr->adev;340 struct amdgpu_device *adev = xcp_mgr->adev;
436- int num_xcc, num_xccs_per_xcp;341+ int num_xcc, num_xccs_per_xcp, r;
342+ int num_xcp, nps_mode;
343+ u16 supp_nps_modes;
344+ bool comp_mode;
437 345 
346+ nps_mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
347+ r = __aqua_vanjaram_get_px_mode_info(xcp_mgr, mode, &num_xcp,
348+ &supp_nps_modes);
349+ if (r)
350+ return false;
351+ 
352+ comp_mode = !!(BIT(nps_mode) & supp_nps_modes);
438 num_xcc = NUM_XCC(adev->gfx.xcc_mask);353 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
439 switch (mode) {354 switch (mode) {
440 case AMDGPU_SPX_PARTITION_MODE:355 case AMDGPU_SPX_PARTITION_MODE:
441- return adev->gmc.num_mem_partitions == 1 && num_xcc > 0;356+ return comp_mode && num_xcc > 0;
442 case AMDGPU_DPX_PARTITION_MODE:357 case AMDGPU_DPX_PARTITION_MODE:
443- return adev->gmc.num_mem_partitions <= 2 && (num_xcc % 4) == 0;358+ return comp_mode && (num_xcc % 4) == 0;
444 case AMDGPU_TPX_PARTITION_MODE:359 case AMDGPU_TPX_PARTITION_MODE:
445- return (adev->gmc.num_mem_partitions == 1 ||360+ return comp_mode && ((num_xcc % 3) == 0);
446- adev->gmc.num_mem_partitions == 3) &&
447- ((num_xcc % 3) == 0);
448 case AMDGPU_QPX_PARTITION_MODE:361 case AMDGPU_QPX_PARTITION_MODE:
449 num_xccs_per_xcp = num_xcc / 4;362 num_xccs_per_xcp = num_xcc / 4;
450- return (adev->gmc.num_mem_partitions == 1 ||363+ return comp_mode && (num_xccs_per_xcp >= 2);
451- adev->gmc.num_mem_partitions == 4) &&
452- (num_xccs_per_xcp >= 2);
453 case AMDGPU_CPX_PARTITION_MODE:364 case AMDGPU_CPX_PARTITION_MODE:
454- return ((num_xcc > 1) &&365+ return comp_mode && (num_xcc > 1);
455- (adev->gmc.num_mem_partitions == 1 || adev->gmc.num_mem_partitions == 4) &&
456- (num_xcc % adev->gmc.num_mem_partitions) == 0);
457 default:366 default:
458 return false;367 return false;
459 }368 }
@@ -461,31 +370,16 @@ static bool __aqua_vanjaram_is_valid_mode(struct amdgpu_xcp_mgr *xcp_mgr,
461 return false;370 return false;
462}371}
463 372 
464-static int __aqua_vanjaram_pre_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags)373+static void __aqua_vanjaram_update_available_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr)
465{374{
466- /* TODO:375+ int mode;
467- * Stop user queues and threads, and make sure GPU is empty of work.
468- */
469 376 
470- if (flags & AMDGPU_XCP_OPS_KFD)377+ xcp_mgr->avail_xcp_modes = 0;
471- amdgpu_amdkfd_device_fini_sw(xcp_mgr->adev);
472 378 
473- return 0;379+ for_each_inst(mode, xcp_mgr->supp_xcp_modes) {
474-}380+ if (__aqua_vanjaram_is_valid_mode(xcp_mgr, mode))
475- 381+ xcp_mgr->avail_xcp_modes |= BIT(mode);
476-static int __aqua_vanjaram_post_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags)
477-{
478- int ret = 0;
479- 
480- if (flags & AMDGPU_XCP_OPS_KFD) {
481- amdgpu_amdkfd_device_probe(xcp_mgr->adev);
482- amdgpu_amdkfd_device_init(xcp_mgr->adev);
483- /* If KFD init failed, return failure */
484- if (!xcp_mgr->adev->kfd.init_complete)
485- ret = -EIO;
486 }382 }
487- 
488- return ret;
489}383}
490 384 
491static int aqua_vanjaram_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,385static int aqua_vanjaram_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
@@ -513,7 +407,8 @@ static int aqua_vanjaram_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
513 return -EINVAL;407 return -EINVAL;
514 }408 }
515 409 
516- if (adev->kfd.init_complete)410+ if (adev->kfd.init_complete && !amdgpu_in_reset(adev) &&
411+ !adev->in_suspend)
517 flags |= AMDGPU_XCP_OPS_KFD;412 flags |= AMDGPU_XCP_OPS_KFD;
518 413 
519 if (flags & AMDGPU_XCP_OPS_KFD) {414 if (flags & AMDGPU_XCP_OPS_KFD) {
@@ -522,7 +417,7 @@ static int aqua_vanjaram_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
522 goto out;417 goto out;
523 }418 }
524 419 
525- ret = __aqua_vanjaram_pre_partition_switch(xcp_mgr, flags);420+ ret = amdgpu_xcp_pre_partition_switch(xcp_mgr, flags);
526 if (ret)421 if (ret)
527 goto unlock;422 goto unlock;
528 423 
@@ -535,7 +430,9 @@ static int aqua_vanjaram_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
535 *num_xcps = num_xcc / num_xcc_per_xcp;430 *num_xcps = num_xcc / num_xcc_per_xcp;
536 amdgpu_xcp_init(xcp_mgr, *num_xcps, mode);431 amdgpu_xcp_init(xcp_mgr, *num_xcps, mode);
537 432 
538- ret = __aqua_vanjaram_post_partition_switch(xcp_mgr, flags);433+ ret = amdgpu_xcp_post_partition_switch(xcp_mgr, flags);
434+ if (!ret)
435+ __aqua_vanjaram_update_available_partition_mode(xcp_mgr);
539unlock:436unlock:
540 if (flags & AMDGPU_XCP_OPS_KFD)437 if (flags & AMDGPU_XCP_OPS_KFD)
541 amdgpu_amdkfd_unlock_kfd(adev);438 amdgpu_amdkfd_unlock_kfd(adev);
@@ -614,20 +511,23 @@ struct amdgpu_xcp_mgr_funcs aqua_vanjaram_xcp_funcs = {
614 .switch_partition_mode = &aqua_vanjaram_switch_partition_mode,511 .switch_partition_mode = &aqua_vanjaram_switch_partition_mode,
615 .query_partition_mode = &aqua_vanjaram_query_partition_mode,512 .query_partition_mode = &aqua_vanjaram_query_partition_mode,
616 .get_ip_details = &aqua_vanjaram_get_xcp_ip_details,513 .get_ip_details = &aqua_vanjaram_get_xcp_ip_details,
514+ .get_xcp_res_info = &aqua_vanjaram_get_xcp_res_info,
617 .get_xcp_mem_id = &aqua_vanjaram_get_xcp_mem_id,515 .get_xcp_mem_id = &aqua_vanjaram_get_xcp_mem_id,
618- .select_scheds = &aqua_vanjaram_select_scheds,
619- .update_partition_sched_list = &aqua_vanjaram_update_partition_sched_list
620};516};
621 517 
622static int aqua_vanjaram_xcp_mgr_init(struct amdgpu_device *adev)518static int aqua_vanjaram_xcp_mgr_init(struct amdgpu_device *adev)
623{519{
624 int ret;520 int ret;
625 521 
522+ if (amdgpu_sriov_vf(adev))
523+ aqua_vanjaram_xcp_funcs.switch_partition_mode = NULL;
524+ 
626 ret = amdgpu_xcp_mgr_init(adev, AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE, 1,525 ret = amdgpu_xcp_mgr_init(adev, AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE, 1,
627 &aqua_vanjaram_xcp_funcs);526 &aqua_vanjaram_xcp_funcs);
628 if (ret)527 if (ret)
629 return ret;528 return ret;
630 529 
530+ amdgpu_xcp_update_supported_modes(adev->xcp_mgr);
631 /* TODO: Default memory node affinity init */531 /* TODO: Default memory node affinity init */
632 532 
633 return ret;533 return ret;
@@ -635,7 +535,7 @@ static int aqua_vanjaram_xcp_mgr_init(struct amdgpu_device *adev)
635 535 
636int aqua_vanjaram_init_soc_config(struct amdgpu_device *adev)536int aqua_vanjaram_init_soc_config(struct amdgpu_device *adev)
637{537{
638- u32 mask, inst_mask = adev->sdma.sdma_mask;538+ u32 mask, avail_inst, inst_mask = adev->sdma.sdma_mask;
639 int ret, i;539 int ret, i;
640 540 
641 /* generally 1 AID supports 4 instances */541 /* generally 1 AID supports 4 instances */
@@ -647,7 +547,9 @@ int aqua_vanjaram_init_soc_config(struct amdgpu_device *adev)
647 547 
648 for (mask = (1 << adev->sdma.num_inst_per_aid) - 1; inst_mask;548 for (mask = (1 << adev->sdma.num_inst_per_aid) - 1; inst_mask;
649 inst_mask >>= adev->sdma.num_inst_per_aid, ++i) {549 inst_mask >>= adev->sdma.num_inst_per_aid, ++i) {
650- if ((inst_mask & mask) == mask)550+ avail_inst = inst_mask & mask;
551+ if (avail_inst == mask || avail_inst == 0x3 ||
552+ avail_inst == 0xc)
651 adev->aid_mask |= (1 << i);553 adev->aid_mask |= (1 << i);
652 }554 }
653 555 
@@ -665,7 +567,420 @@ int aqua_vanjaram_init_soc_config(struct amdgpu_device *adev)
665 if (ret)567 if (ret)
666 return ret;568 return ret;
667 569 
668- aqua_vanjaram_ip_map_init(adev);570+ amdgpu_ip_map_init(adev);
669 571 
670 return 0;572 return 0;
671}573}
574+ 
575+static void aqua_read_smn(struct amdgpu_device *adev,
576+ struct amdgpu_smn_reg_data *regdata,
577+ uint64_t smn_addr)
578+{
579+ regdata->addr = smn_addr;
580+ regdata->value = RREG32_PCIE(smn_addr);
581+}
582+ 
583+struct aqua_reg_list {
584+ uint64_t start_addr;
585+ uint32_t num_regs;
586+ uint32_t incrx;
587+};
588+ 
589+#define DW_ADDR_INCR 4
590+ 
591+static void aqua_read_smn_ext(struct amdgpu_device *adev,
592+ struct amdgpu_smn_reg_data *regdata,
593+ uint64_t smn_addr, int i)
594+{
595+ regdata->addr =
596+ smn_addr + adev->asic_funcs->encode_ext_smn_addressing(i);
597+ regdata->value = RREG32_PCIE_EXT(regdata->addr);
598+}
599+ 
600+#define smnreg_0x1A340218 0x1A340218
601+#define smnreg_0x1A3402E4 0x1A3402E4
602+#define smnreg_0x1A340294 0x1A340294
603+#define smreg_0x1A380088 0x1A380088
604+ 
605+#define NUM_PCIE_SMN_REGS 14
606+ 
607+static struct aqua_reg_list pcie_reg_addrs[] = {
608+ { smnreg_0x1A340218, 1, 0 },
609+ { smnreg_0x1A3402E4, 1, 0 },
610+ { smnreg_0x1A340294, 6, DW_ADDR_INCR },
611+ { smreg_0x1A380088, 6, DW_ADDR_INCR },
612+};
613+ 
614+static ssize_t aqua_vanjaram_read_pcie_state(struct amdgpu_device *adev,
615+ void *buf, size_t max_size)
616+{
617+ struct amdgpu_reg_state_pcie_v1_0 *pcie_reg_state;
618+ uint32_t start_addr, incrx, num_regs, szbuf;
619+ struct amdgpu_regs_pcie_v1_0 *pcie_regs;
620+ struct amdgpu_smn_reg_data *reg_data;
621+ struct pci_dev *us_pdev, *ds_pdev;
622+ int aer_cap, r, n;
623+ 
624+ if (!buf || !max_size)
625+ return -EINVAL;
626+ 
627+ pcie_reg_state = (struct amdgpu_reg_state_pcie_v1_0 *)buf;
628+ 
629+ szbuf = sizeof(*pcie_reg_state) +
630+ amdgpu_reginst_size(1, sizeof(*pcie_regs), NUM_PCIE_SMN_REGS);
631+ /* Only one instance of pcie regs */
632+ if (max_size < szbuf)
633+ return -EOVERFLOW;
634+ 
635+ pcie_regs = (struct amdgpu_regs_pcie_v1_0 *)((uint8_t *)buf +
636+ sizeof(*pcie_reg_state));
637+ pcie_regs->inst_header.instance = 0;
638+ pcie_regs->inst_header.state = AMDGPU_INST_S_OK;
639+ pcie_regs->inst_header.num_smn_regs = NUM_PCIE_SMN_REGS;
640+ 
641+ reg_data = pcie_regs->smn_reg_values;
642+ 
643+ for (r = 0; r < ARRAY_SIZE(pcie_reg_addrs); r++) {
644+ start_addr = pcie_reg_addrs[r].start_addr;
645+ incrx = pcie_reg_addrs[r].incrx;
646+ num_regs = pcie_reg_addrs[r].num_regs;
647+ for (n = 0; n < num_regs; n++) {
648+ aqua_read_smn(adev, reg_data, start_addr + n * incrx);
649+ ++reg_data;
650+ }
651+ }
652+ 
653+ ds_pdev = pci_upstream_bridge(adev->pdev);
654+ us_pdev = pci_upstream_bridge(ds_pdev);
655+ 
656+ pcie_capability_read_word(us_pdev, PCI_EXP_DEVSTA,
657+ &pcie_regs->device_status);
658+ pcie_capability_read_word(us_pdev, PCI_EXP_LNKSTA,
659+ &pcie_regs->link_status);
660+ 
661+ aer_cap = pci_find_ext_capability(us_pdev, PCI_EXT_CAP_ID_ERR);
662+ if (aer_cap) {
663+ pci_read_config_dword(us_pdev, aer_cap + PCI_ERR_COR_STATUS,
664+ &pcie_regs->pcie_corr_err_status);
665+ pci_read_config_dword(us_pdev, aer_cap + PCI_ERR_UNCOR_STATUS,
666+ &pcie_regs->pcie_uncorr_err_status);
667+ }
668+ 
669+ pci_read_config_dword(us_pdev, PCI_PRIMARY_BUS,
670+ &pcie_regs->sub_bus_number_latency);
671+ 
672+ pcie_reg_state->common_header.structure_size = szbuf;
673+ pcie_reg_state->common_header.format_revision = 1;
674+ pcie_reg_state->common_header.content_revision = 0;
675+ pcie_reg_state->common_header.state_type = AMDGPU_REG_STATE_TYPE_PCIE;
676+ pcie_reg_state->common_header.num_instances = 1;
677+ 
678+ return pcie_reg_state->common_header.structure_size;
679+}
680+ 
681+#define smnreg_0x11A00050 0x11A00050
682+#define smnreg_0x11A00180 0x11A00180
683+#define smnreg_0x11A00070 0x11A00070
684+#define smnreg_0x11A00200 0x11A00200
685+#define smnreg_0x11A0020C 0x11A0020C
686+#define smnreg_0x11A00210 0x11A00210
687+#define smnreg_0x11A00108 0x11A00108
688+ 
689+#define XGMI_LINK_REG(smnreg, l) ((smnreg) | (l << 20))
690+ 
691+#define NUM_XGMI_SMN_REGS 25
692+ 
693+static struct aqua_reg_list xgmi_reg_addrs[] = {
694+ { smnreg_0x11A00050, 1, 0 },
695+ { smnreg_0x11A00180, 16, DW_ADDR_INCR },
696+ { smnreg_0x11A00070, 4, DW_ADDR_INCR },
697+ { smnreg_0x11A00200, 1, 0 },
698+ { smnreg_0x11A0020C, 1, 0 },
699+ { smnreg_0x11A00210, 1, 0 },
700+ { smnreg_0x11A00108, 1, 0 },
701+};
702+ 
703+static ssize_t aqua_vanjaram_read_xgmi_state(struct amdgpu_device *adev,
704+ void *buf, size_t max_size)
705+{
706+ struct amdgpu_reg_state_xgmi_v1_0 *xgmi_reg_state;
707+ uint32_t start_addr, incrx, num_regs, szbuf;
708+ struct amdgpu_regs_xgmi_v1_0 *xgmi_regs;
709+ struct amdgpu_smn_reg_data *reg_data;
710+ const int max_xgmi_instances = 8;
711+ int inst = 0, i, j, r, n;
712+ const int xgmi_inst = 2;
713+ void *p;
714+ 
715+ if (!buf || !max_size)
716+ return -EINVAL;
717+ 
718+ xgmi_reg_state = (struct amdgpu_reg_state_xgmi_v1_0 *)buf;
719+ 
720+ szbuf = sizeof(*xgmi_reg_state) +
721+ amdgpu_reginst_size(max_xgmi_instances, sizeof(*xgmi_regs),
722+ NUM_XGMI_SMN_REGS);
723+ /* Only one instance of pcie regs */
724+ if (max_size < szbuf)
725+ return -EOVERFLOW;
726+ 
727+ p = &xgmi_reg_state->xgmi_state_regs[0];
728+ for_each_inst(i, adev->aid_mask) {
729+ for (j = 0; j < xgmi_inst; ++j) {
730+ xgmi_regs = (struct amdgpu_regs_xgmi_v1_0 *)p;
731+ xgmi_regs->inst_header.instance = inst++;
732+ 
733+ xgmi_regs->inst_header.state = AMDGPU_INST_S_OK;
734+ xgmi_regs->inst_header.num_smn_regs = NUM_XGMI_SMN_REGS;
735+ 
736+ reg_data = xgmi_regs->smn_reg_values;
737+ 
738+ for (r = 0; r < ARRAY_SIZE(xgmi_reg_addrs); r++) {
739+ start_addr = xgmi_reg_addrs[r].start_addr;
740+ incrx = xgmi_reg_addrs[r].incrx;
741+ num_regs = xgmi_reg_addrs[r].num_regs;
742+ 
743+ for (n = 0; n < num_regs; n++) {
744+ aqua_read_smn_ext(
745+ adev, reg_data,
746+ XGMI_LINK_REG(start_addr, j) +
747+ n * incrx,
748+ i);
749+ ++reg_data;
750+ }
751+ }
752+ p = reg_data;
753+ }
754+ }
755+ 
756+ xgmi_reg_state->common_header.structure_size = szbuf;
757+ xgmi_reg_state->common_header.format_revision = 1;
758+ xgmi_reg_state->common_header.content_revision = 0;
759+ xgmi_reg_state->common_header.state_type = AMDGPU_REG_STATE_TYPE_XGMI;
760+ xgmi_reg_state->common_header.num_instances = max_xgmi_instances;
761+ 
762+ return xgmi_reg_state->common_header.structure_size;
763+}
764+ 
765+#define smnreg_0x11C00070 0x11C00070
766+#define smnreg_0x11C00210 0x11C00210
767+ 
768+static struct aqua_reg_list wafl_reg_addrs[] = {
769+ { smnreg_0x11C00070, 4, DW_ADDR_INCR },
770+ { smnreg_0x11C00210, 1, 0 },
771+};
772+ 
773+#define WAFL_LINK_REG(smnreg, l) ((smnreg) | (l << 20))
774+ 
775+#define NUM_WAFL_SMN_REGS 5
776+ 
777+static ssize_t aqua_vanjaram_read_wafl_state(struct amdgpu_device *adev,
778+ void *buf, size_t max_size)
779+{
780+ struct amdgpu_reg_state_wafl_v1_0 *wafl_reg_state;
781+ uint32_t start_addr, incrx, num_regs, szbuf;
782+ struct amdgpu_regs_wafl_v1_0 *wafl_regs;
783+ struct amdgpu_smn_reg_data *reg_data;
784+ const int max_wafl_instances = 8;
785+ int inst = 0, i, j, r, n;
786+ const int wafl_inst = 2;
787+ void *p;
788+ 
789+ if (!buf || !max_size)
790+ return -EINVAL;
791+ 
792+ wafl_reg_state = (struct amdgpu_reg_state_wafl_v1_0 *)buf;
793+ 
794+ szbuf = sizeof(*wafl_reg_state) +
795+ amdgpu_reginst_size(max_wafl_instances, sizeof(*wafl_regs),
796+ NUM_WAFL_SMN_REGS);
797+ 
798+ if (max_size < szbuf)
799+ return -EOVERFLOW;
800+ 
801+ p = &wafl_reg_state->wafl_state_regs[0];
802+ for_each_inst(i, adev->aid_mask) {
803+ for (j = 0; j < wafl_inst; ++j) {
804+ wafl_regs = (struct amdgpu_regs_wafl_v1_0 *)p;
805+ wafl_regs->inst_header.instance = inst++;
806+ 
807+ wafl_regs->inst_header.state = AMDGPU_INST_S_OK;
808+ wafl_regs->inst_header.num_smn_regs = NUM_WAFL_SMN_REGS;
809+ 
810+ reg_data = wafl_regs->smn_reg_values;
811+ 
812+ for (r = 0; r < ARRAY_SIZE(wafl_reg_addrs); r++) {
813+ start_addr = wafl_reg_addrs[r].start_addr;
814+ incrx = wafl_reg_addrs[r].incrx;
815+ num_regs = wafl_reg_addrs[r].num_regs;
816+ for (n = 0; n < num_regs; n++) {
817+ aqua_read_smn_ext(
818+ adev, reg_data,
819+ WAFL_LINK_REG(start_addr, j) +
820+ n * incrx,
821+ i);
822+ ++reg_data;
823+ }
824+ }
825+ p = reg_data;
826+ }
827+ }
828+ 
829+ wafl_reg_state->common_header.structure_size = szbuf;
830+ wafl_reg_state->common_header.format_revision = 1;
831+ wafl_reg_state->common_header.content_revision = 0;
832+ wafl_reg_state->common_header.state_type = AMDGPU_REG_STATE_TYPE_WAFL;
833+ wafl_reg_state->common_header.num_instances = max_wafl_instances;
834+ 
835+ return wafl_reg_state->common_header.structure_size;
836+}
837+ 
838+#define smnreg_0x1B311060 0x1B311060
839+#define smnreg_0x1B411060 0x1B411060
840+#define smnreg_0x1B511060 0x1B511060
841+#define smnreg_0x1B611060 0x1B611060
842+ 
843+#define smnreg_0x1C307120 0x1C307120
844+#define smnreg_0x1C317120 0x1C317120
845+ 
846+#define smnreg_0x1C320830 0x1C320830
847+#define smnreg_0x1C380830 0x1C380830
848+#define smnreg_0x1C3D0830 0x1C3D0830
849+#define smnreg_0x1C420830 0x1C420830
850+ 
851+#define smnreg_0x1C320100 0x1C320100
852+#define smnreg_0x1C380100 0x1C380100
853+#define smnreg_0x1C3D0100 0x1C3D0100
854+#define smnreg_0x1C420100 0x1C420100
855+ 
856+#define smnreg_0x1B310500 0x1B310500
857+#define smnreg_0x1C300400 0x1C300400
858+ 
859+#define USR_CAKE_INCR 0x11000
860+#define USR_LINK_INCR 0x100000
861+#define USR_CP_INCR 0x10000
862+ 
863+#define NUM_USR_SMN_REGS 20
864+ 
865+struct aqua_reg_list usr_reg_addrs[] = {
866+ { smnreg_0x1B311060, 4, DW_ADDR_INCR },
867+ { smnreg_0x1B411060, 4, DW_ADDR_INCR },
868+ { smnreg_0x1B511060, 4, DW_ADDR_INCR },
869+ { smnreg_0x1B611060, 4, DW_ADDR_INCR },
870+ { smnreg_0x1C307120, 2, DW_ADDR_INCR },
871+ { smnreg_0x1C317120, 2, DW_ADDR_INCR },
872+};
873+ 
874+#define NUM_USR1_SMN_REGS 46
875+struct aqua_reg_list usr1_reg_addrs[] = {
876+ { smnreg_0x1C320830, 6, USR_CAKE_INCR },
877+ { smnreg_0x1C380830, 5, USR_CAKE_INCR },
878+ { smnreg_0x1C3D0830, 5, USR_CAKE_INCR },
879+ { smnreg_0x1C420830, 4, USR_CAKE_INCR },
880+ { smnreg_0x1C320100, 6, USR_CAKE_INCR },
881+ { smnreg_0x1C380100, 5, USR_CAKE_INCR },
882+ { smnreg_0x1C3D0100, 5, USR_CAKE_INCR },
883+ { smnreg_0x1C420100, 4, USR_CAKE_INCR },
884+ { smnreg_0x1B310500, 4, USR_LINK_INCR },
885+ { smnreg_0x1C300400, 2, USR_CP_INCR },
886+};
887+ 
888+static ssize_t aqua_vanjaram_read_usr_state(struct amdgpu_device *adev,
889+ void *buf, size_t max_size,
890+ int reg_state)
891+{
892+ uint32_t start_addr, incrx, num_regs, szbuf, num_smn;
893+ struct amdgpu_reg_state_usr_v1_0 *usr_reg_state;
894+ struct amdgpu_regs_usr_v1_0 *usr_regs;
895+ struct amdgpu_smn_reg_data *reg_data;
896+ const int max_usr_instances = 4;
897+ struct aqua_reg_list *reg_addrs;
898+ int inst = 0, i, n, r, arr_size;
899+ void *p;
900+ 
901+ if (!buf || !max_size)
902+ return -EINVAL;
903+ 
904+ switch (reg_state) {
905+ case AMDGPU_REG_STATE_TYPE_USR:
906+ arr_size = ARRAY_SIZE(usr_reg_addrs);
907+ reg_addrs = usr_reg_addrs;
908+ num_smn = NUM_USR_SMN_REGS;
909+ break;
910+ case AMDGPU_REG_STATE_TYPE_USR_1:
911+ arr_size = ARRAY_SIZE(usr1_reg_addrs);
912+ reg_addrs = usr1_reg_addrs;
913+ num_smn = NUM_USR1_SMN_REGS;
914+ break;
915+ default:
916+ return -EINVAL;
917+ }
918+ 
919+ usr_reg_state = (struct amdgpu_reg_state_usr_v1_0 *)buf;
920+ 
921+ szbuf = sizeof(*usr_reg_state) + amdgpu_reginst_size(max_usr_instances,
922+ sizeof(*usr_regs),
923+ num_smn);
924+ if (max_size < szbuf)
925+ return -EOVERFLOW;
926+ 
927+ p = &usr_reg_state->usr_state_regs[0];
928+ for_each_inst(i, adev->aid_mask) {
929+ usr_regs = (struct amdgpu_regs_usr_v1_0 *)p;
930+ usr_regs->inst_header.instance = inst++;
931+ usr_regs->inst_header.state = AMDGPU_INST_S_OK;
932+ usr_regs->inst_header.num_smn_regs = num_smn;
933+ reg_data = usr_regs->smn_reg_values;
934+ 
935+ for (r = 0; r < arr_size; r++) {
936+ start_addr = reg_addrs[r].start_addr;
937+ incrx = reg_addrs[r].incrx;
938+ num_regs = reg_addrs[r].num_regs;
939+ for (n = 0; n < num_regs; n++) {
940+ aqua_read_smn_ext(adev, reg_data,
941+ start_addr + n * incrx, i);
942+ reg_data++;
943+ }
944+ }
945+ p = reg_data;
946+ }
947+ 
948+ usr_reg_state->common_header.structure_size = szbuf;
949+ usr_reg_state->common_header.format_revision = 1;
950+ usr_reg_state->common_header.content_revision = 0;
951+ usr_reg_state->common_header.state_type = AMDGPU_REG_STATE_TYPE_USR;
952+ usr_reg_state->common_header.num_instances = max_usr_instances;
953+ 
954+ return usr_reg_state->common_header.structure_size;
955+}
956+ 
957+ssize_t aqua_vanjaram_get_reg_state(struct amdgpu_device *adev,
958+ enum amdgpu_reg_state reg_state, void *buf,
959+ size_t max_size)
960+{
961+ ssize_t size;
962+ 
963+ switch (reg_state) {
964+ case AMDGPU_REG_STATE_TYPE_PCIE:
965+ size = aqua_vanjaram_read_pcie_state(adev, buf, max_size);
966+ break;
967+ case AMDGPU_REG_STATE_TYPE_XGMI:
968+ size = aqua_vanjaram_read_xgmi_state(adev, buf, max_size);
969+ break;
970+ case AMDGPU_REG_STATE_TYPE_WAFL:
971+ size = aqua_vanjaram_read_wafl_state(adev, buf, max_size);
972+ break;
973+ case AMDGPU_REG_STATE_TYPE_USR:
974+ size = aqua_vanjaram_read_usr_state(adev, buf, max_size,
975+ AMDGPU_REG_STATE_TYPE_USR);
976+ break;
977+ case AMDGPU_REG_STATE_TYPE_USR_1:
978+ size = aqua_vanjaram_read_usr_state(
979+ adev, buf, max_size, AMDGPU_REG_STATE_TYPE_USR_1);
980+ break;
981+ default:
982+ return -EINVAL;
983+ }
984+ 
985+ return size;
986+}
Mdrivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c+1093-463
@@ -37,10 +37,18 @@
37#include "gc/gc_9_4_3_sh_mask.h"37#include "gc/gc_9_4_3_sh_mask.h"
38 38 
39#include "gfx_v9_4_3.h"39#include "gfx_v9_4_3.h"
40+#include "gfx_v9_4_3_cleaner_shader.h"
40#include "amdgpu_xcp.h"41#include "amdgpu_xcp.h"
42+#include "amdgpu_aca.h"
41 43 
42MODULE_FIRMWARE("amdgpu/gc_9_4_3_mec.bin");44MODULE_FIRMWARE("amdgpu/gc_9_4_3_mec.bin");
45+MODULE_FIRMWARE("amdgpu/gc_9_4_4_mec.bin");
46+MODULE_FIRMWARE("amdgpu/gc_9_5_0_mec.bin");
43MODULE_FIRMWARE("amdgpu/gc_9_4_3_rlc.bin");47MODULE_FIRMWARE("amdgpu/gc_9_4_3_rlc.bin");
48+MODULE_FIRMWARE("amdgpu/gc_9_4_4_rlc.bin");
49+MODULE_FIRMWARE("amdgpu/gc_9_5_0_rlc.bin");
50+MODULE_FIRMWARE("amdgpu/gc_9_4_3_sjt_mec.bin");
51+MODULE_FIRMWARE("amdgpu/gc_9_4_4_sjt_mec.bin");
44 52 
45#define GFX9_MEC_HPD_SIZE 409653#define GFX9_MEC_HPD_SIZE 4096
46#define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L54#define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
@@ -48,6 +56,111 @@ MODULE_FIRMWARE("amdgpu/gc_9_4_3_rlc.bin");
48#define GOLDEN_GB_ADDR_CONFIG 0x2a11404256#define GOLDEN_GB_ADDR_CONFIG 0x2a114042
49#define CP_HQD_PERSISTENT_STATE_DEFAULT 0xbe0530157#define CP_HQD_PERSISTENT_STATE_DEFAULT 0xbe05301
50 58 
59+#define XCC_REG_RANGE_0_LOW 0x2000 /* XCC gfxdec0 lower Bound */
60+#define XCC_REG_RANGE_0_HIGH 0x3400 /* XCC gfxdec0 upper Bound */
61+#define XCC_REG_RANGE_1_LOW 0xA000 /* XCC gfxdec1 lower Bound */
62+#define XCC_REG_RANGE_1_HIGH 0x10000 /* XCC gfxdec1 upper Bound */
63+ 
64+#define NORMALIZE_XCC_REG_OFFSET(offset) \
65+ (offset & 0xFFFF)
66+ 
67+static const struct amdgpu_hwip_reg_entry gc_reg_list_9_4_3[] = {
68+ SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS),
69+ SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
70+ SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1),
71+ SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2),
72+ SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1),
73+ SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1),
74+ SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT),
75+ SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT),
76+ SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT),
77+ SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS),
78+ SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR),
79+ SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS),
80+ SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS),
81+ SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS),
82+ SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT),
83+ SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT),
84+ SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS),
85+ SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS),
86+ SOC15_REG_ENTRY_STR(GC, 0, regSQC_DCACHE_UTCL1_STATUS),
87+ SOC15_REG_ENTRY_STR(GC, 0, regSQC_ICACHE_UTCL1_STATUS),
88+ SOC15_REG_ENTRY_STR(GC, 0, regSQ_UTCL1_STATUS),
89+ SOC15_REG_ENTRY_STR(GC, 0, regTCP_UTCL1_STATUS),
90+ SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS),
91+ SOC15_REG_ENTRY_STR(GC, 0, regVM_L2_PROTECTION_FAULT_CNTL),
92+ SOC15_REG_ENTRY_STR(GC, 0, regVM_L2_PROTECTION_FAULT_STATUS),
93+ SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG),
94+ SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL),
95+ SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR),
96+ SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC2_INSTR_PNTR),
97+ SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS),
98+ SOC15_REG_ENTRY_STR(GC, 0, regRLC_STAT),
99+ SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_COMMAND),
100+ SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_MESSAGE),
101+ SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_ARGUMENT_1),
102+ SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_ARGUMENT_2),
103+ SOC15_REG_ENTRY_STR(GC, 0, regSMU_RLC_RESPONSE),
104+ SOC15_REG_ENTRY_STR(GC, 0, regRLC_SAFE_MODE),
105+ SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_SAFE_MODE),
106+ SOC15_REG_ENTRY_STR(GC, 0, regRLC_INT_STAT),
107+ SOC15_REG_ENTRY_STR(GC, 0, regRLC_GPM_GENERAL_6),
108+ /* SE status registers */
109+ SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0),
110+ SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1),
111+ SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2),
112+ SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3)
113+};
114+ 
115+static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_9_4_3[] = {
116+ /* compute queue registers */
117+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID),
118+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ACTIVE),
119+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE),
120+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY),
121+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY),
122+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM),
123+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE),
124+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI),
125+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR),
126+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR),
127+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI),
128+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL),
129+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL),
130+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR),
131+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI),
132+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR),
133+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL),
134+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST),
135+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR),
136+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI),
137+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL),
138+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR),
139+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR),
140+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS),
141+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO),
142+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI),
143+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL),
144+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET),
145+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE),
146+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET),
147+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE),
148+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE),
149+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR),
150+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM),
151+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO),
152+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI),
153+ SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GFX_STATUS),
154+ SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
155+ SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
156+ SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
157+ SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
158+ SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
159+ SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
160+ SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
161+ SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
162+};
163+ 
51struct amdgpu_gfx_ras gfx_v9_4_3_ras;164struct amdgpu_gfx_ras gfx_v9_4_3_ras;
52 165 
53static void gfx_v9_4_3_set_ring_funcs(struct amdgpu_device *adev);166static void gfx_v9_4_3_set_ring_funcs(struct amdgpu_device *adev);
@@ -56,10 +169,18 @@ static void gfx_v9_4_3_set_gds_init(struct amdgpu_device *adev);
56static void gfx_v9_4_3_set_rlc_funcs(struct amdgpu_device *adev);169static void gfx_v9_4_3_set_rlc_funcs(struct amdgpu_device *adev);
57static int gfx_v9_4_3_get_cu_info(struct amdgpu_device *adev,170static int gfx_v9_4_3_get_cu_info(struct amdgpu_device *adev,
58 struct amdgpu_cu_info *cu_info);171 struct amdgpu_cu_info *cu_info);
172+static void gfx_v9_4_3_xcc_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
173+static void gfx_v9_4_3_xcc_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
59 174 
60static void gfx_v9_4_3_kiq_set_resources(struct amdgpu_ring *kiq_ring,175static void gfx_v9_4_3_kiq_set_resources(struct amdgpu_ring *kiq_ring,
61 uint64_t queue_mask)176 uint64_t queue_mask)
62{177{
178+ struct amdgpu_device *adev = kiq_ring->adev;
179+ u64 shader_mc_addr;
180+ 
181+ /* Cleaner shader MC address */
182+ shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;
183+ 
63 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));184 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
64 amdgpu_ring_write(kiq_ring,185 amdgpu_ring_write(kiq_ring,
65 PACKET3_SET_RESOURCES_VMID_MASK(0) |186 PACKET3_SET_RESOURCES_VMID_MASK(0) |
@@ -69,8 +190,8 @@ static void gfx_v9_4_3_kiq_set_resources(struct amdgpu_ring *kiq_ring,
69 lower_32_bits(queue_mask)); /* queue mask lo */190 lower_32_bits(queue_mask)); /* queue mask lo */
70 amdgpu_ring_write(kiq_ring,191 amdgpu_ring_write(kiq_ring,
71 upper_32_bits(queue_mask)); /* queue mask hi */192 upper_32_bits(queue_mask)); /* queue mask hi */
72- amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */193+ amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */
73- amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */194+ amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */
74 amdgpu_ring_write(kiq_ring, 0); /* oac mask */195 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
75 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */196 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
76}197}
@@ -167,12 +288,46 @@ static void gfx_v9_4_3_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
167 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));288 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
168}289}
169 290 
291+static void gfx_v9_4_3_kiq_reset_hw_queue(struct amdgpu_ring *kiq_ring, uint32_t queue_type,
292+ uint32_t me_id, uint32_t pipe_id, uint32_t queue_id,
293+ uint32_t xcc_id, uint32_t vmid)
294+{
295+ struct amdgpu_device *adev = kiq_ring->adev;
296+ unsigned i;
297+ 
298+ /* enter save mode */
299+ amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
300+ mutex_lock(&adev->srbm_mutex);
301+ soc15_grbm_select(adev, me_id, pipe_id, queue_id, 0, xcc_id);
302+ 
303+ if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
304+ WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 0x2);
305+ WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_COMPUTE_QUEUE_RESET, 0x1);
306+ /* wait till dequeue take effects */
307+ for (i = 0; i < adev->usec_timeout; i++) {
308+ if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
309+ break;
310+ udelay(1);
311+ }
312+ if (i >= adev->usec_timeout)
313+ dev_err(adev->dev, "fail to wait on hqd deactive\n");
314+ } else {
315+ dev_err(adev->dev, "reset queue_type(%d) not supported\n\n", queue_type);
316+ }
317+ 
318+ soc15_grbm_select(adev, 0, 0, 0, 0, 0);
319+ mutex_unlock(&adev->srbm_mutex);
320+ /* exit safe mode */
321+ amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
322+}
323+ 
170static const struct kiq_pm4_funcs gfx_v9_4_3_kiq_pm4_funcs = {324static const struct kiq_pm4_funcs gfx_v9_4_3_kiq_pm4_funcs = {
171 .kiq_set_resources = gfx_v9_4_3_kiq_set_resources,325 .kiq_set_resources = gfx_v9_4_3_kiq_set_resources,
172 .kiq_map_queues = gfx_v9_4_3_kiq_map_queues,326 .kiq_map_queues = gfx_v9_4_3_kiq_map_queues,
173 .kiq_unmap_queues = gfx_v9_4_3_kiq_unmap_queues,327 .kiq_unmap_queues = gfx_v9_4_3_kiq_unmap_queues,
174 .kiq_query_status = gfx_v9_4_3_kiq_query_status,328 .kiq_query_status = gfx_v9_4_3_kiq_query_status,
175 .kiq_invalidate_tlbs = gfx_v9_4_3_kiq_invalidate_tlbs,329 .kiq_invalidate_tlbs = gfx_v9_4_3_kiq_invalidate_tlbs,
330+ .kiq_reset_hw_queue = gfx_v9_4_3_kiq_reset_hw_queue,
176 .set_resources_size = 8,331 .set_resources_size = 8,
177 .map_queues_size = 7,332 .map_queues_size = 7,
178 .unmap_queues_size = 6,333 .unmap_queues_size = 6,
@@ -199,20 +354,28 @@ static void gfx_v9_4_3_init_golden_registers(struct amdgpu_device *adev)
199 354 
200 WREG32_SOC15(GC, dev_inst, regGB_ADDR_CONFIG,355 WREG32_SOC15(GC, dev_inst, regGB_ADDR_CONFIG,
201 GOLDEN_GB_ADDR_CONFIG);356 GOLDEN_GB_ADDR_CONFIG);
202- /* Golden settings applied by driver for ASIC with rev_id 0 */357+ WREG32_FIELD15_PREREG(GC, dev_inst, TCP_UTCL1_CNTL2, SPARE, 0x1);
203- if (adev->rev_id == 0) {
204- WREG32_FIELD15_PREREG(GC, dev_inst, TCP_UTCL1_CNTL1,
205- REDUCE_FIFO_DEPTH_BY_2, 2);
206- } else {
207- WREG32_FIELD15_PREREG(GC, dev_inst, TCP_UTCL1_CNTL2,
208- SPARE, 0x1);
209- }
210 }358 }
211}359}
212 360 
361+static uint32_t gfx_v9_4_3_normalize_xcc_reg_offset(uint32_t reg)
362+{
363+ uint32_t normalized_reg = NORMALIZE_XCC_REG_OFFSET(reg);
364+ 
365+ /* If it is an XCC reg, normalize the reg to keep
366+ lower 16 bits in local xcc */
367+ 
368+ if (((normalized_reg >= XCC_REG_RANGE_0_LOW) && (normalized_reg < XCC_REG_RANGE_0_HIGH)) ||
369+ ((normalized_reg >= XCC_REG_RANGE_1_LOW) && (normalized_reg < XCC_REG_RANGE_1_HIGH)))
370+ return normalized_reg;
371+ else
372+ return reg;
373+}
374+ 
213static void gfx_v9_4_3_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,375static void gfx_v9_4_3_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
214 bool wc, uint32_t reg, uint32_t val)376 bool wc, uint32_t reg, uint32_t val)
215{377{
378+ reg = gfx_v9_4_3_normalize_xcc_reg_offset(reg);
216 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));379 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
217 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |380 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
218 WRITE_DATA_DST_SEL(0) |381 WRITE_DATA_DST_SEL(0) |
@@ -227,6 +390,12 @@ static void gfx_v9_4_3_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
227 uint32_t addr1, uint32_t ref, uint32_t mask,390 uint32_t addr1, uint32_t ref, uint32_t mask,
228 uint32_t inv)391 uint32_t inv)
229{392{
393+ /* Only do the normalization on regspace */
394+ if (mem_space == 0) {
395+ addr0 = gfx_v9_4_3_normalize_xcc_reg_offset(addr0);
396+ addr1 = gfx_v9_4_3_normalize_xcc_reg_offset(addr1);
397+ }
398+ 
230 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));399 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
231 amdgpu_ring_write(ring,400 amdgpu_ring_write(ring,
232 /* memory (1) or register (0) */401 /* memory (1) or register (0) */
@@ -256,6 +425,7 @@ static int gfx_v9_4_3_ring_test_ring(struct amdgpu_ring *ring)
256 xcc_offset = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);425 xcc_offset = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
257 scratch_reg0_offset = SOC15_REG_OFFSET(GC, GET_INST(GC, ring->xcc_id), regSCRATCH_REG0);426 scratch_reg0_offset = SOC15_REG_OFFSET(GC, GET_INST(GC, ring->xcc_id), regSCRATCH_REG0);
258 WREG32(scratch_reg0_offset, 0xCAFEDEAD);427 WREG32(scratch_reg0_offset, 0xCAFEDEAD);
428+ tmp = RREG32(scratch_reg0_offset);
259 429 
260 r = amdgpu_ring_alloc(ring, 3);430 r = amdgpu_ring_alloc(ring, 3);
261 if (r)431 if (r)
@@ -327,7 +497,7 @@ static int gfx_v9_4_3_ring_test_ib(struct amdgpu_ring *ring, long timeout)
327 r = -EINVAL;497 r = -EINVAL;
328 498 
329err2:499err2:
330- amdgpu_ib_free(adev, &ib, NULL);500+ amdgpu_ib_free(&ib, NULL);
331 dma_fence_put(f);501 dma_fence_put(f);
332err1:502err1:
333 amdgpu_device_wb_free(adev, index);503 amdgpu_device_wb_free(adev, index);
@@ -364,15 +534,15 @@ static void gfx_v9_4_3_free_microcode(struct amdgpu_device *adev)
364static int gfx_v9_4_3_init_rlc_microcode(struct amdgpu_device *adev,534static int gfx_v9_4_3_init_rlc_microcode(struct amdgpu_device *adev,
365 const char *chip_name)535 const char *chip_name)
366{536{
367- char fw_name[30];
368 int err;537 int err;
369 const struct rlc_firmware_header_v2_0 *rlc_hdr;538 const struct rlc_firmware_header_v2_0 *rlc_hdr;
370 uint16_t version_major;539 uint16_t version_major;
371 uint16_t version_minor;540 uint16_t version_minor;
372 541 
373- snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
374 542 
375- err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, fw_name);543+ err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
544+ AMDGPU_UCODE_REQUIRED,
545+ "amdgpu/%s_rlc.bin", chip_name);
376 if (err)546 if (err)
377 goto out;547 goto out;
378 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;548 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
@@ -387,26 +557,24 @@ static int gfx_v9_4_3_init_rlc_microcode(struct amdgpu_device *adev,
387 return err;557 return err;
388}558}
389 559 
390-static bool gfx_v9_4_3_should_disable_gfxoff(struct pci_dev *pdev)
391-{
392- return true;
393-}
394- 
395-static void gfx_v9_4_3_check_if_need_gfxoff(struct amdgpu_device *adev)
396-{
397- if (gfx_v9_4_3_should_disable_gfxoff(adev->pdev))
398- adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
399-}
400- 
401static int gfx_v9_4_3_init_cp_compute_microcode(struct amdgpu_device *adev,560static int gfx_v9_4_3_init_cp_compute_microcode(struct amdgpu_device *adev,
402 const char *chip_name)561 const char *chip_name)
403{562{
404- char fw_name[30];
405 int err;563 int err;
406 564 
407- snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);565+ if (amdgpu_sriov_vf(adev)) {
566+ err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
567+ AMDGPU_UCODE_REQUIRED,
568+ "amdgpu/%s_sjt_mec.bin", chip_name);
408 569 
409- err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, fw_name);570+ if (err)
571+ err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
572+ AMDGPU_UCODE_REQUIRED,
573+ "amdgpu/%s_mec.bin", chip_name);
574+ } else
575+ err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
576+ AMDGPU_UCODE_REQUIRED,
577+ "amdgpu/%s_mec.bin", chip_name);
410 if (err)578 if (err)
411 goto out;579 goto out;
412 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);580 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
@@ -415,8 +583,6 @@ static int gfx_v9_4_3_init_cp_compute_microcode(struct amdgpu_device *adev,
415 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;583 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
416 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;584 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
417 585 
418- gfx_v9_4_3_check_if_need_gfxoff(adev);
419- 
420out:586out:
421 if (err)587 if (err)
422 amdgpu_ucode_release(&adev->gfx.mec_fw);588 amdgpu_ucode_release(&adev->gfx.mec_fw);
@@ -618,12 +784,21 @@ static void gfx_v9_4_3_select_me_pipe_q(struct amdgpu_device *adev,
618 soc15_grbm_select(adev, me, pipe, q, vm, GET_INST(GC, xcc_id));784 soc15_grbm_select(adev, me, pipe, q, vm, GET_INST(GC, xcc_id));
619}785}
620 786 
787+static int gfx_v9_4_3_get_xccs_per_xcp(struct amdgpu_device *adev)
788+{
789+ u32 xcp_ctl;
790+ 
791+ /* Value is expected to be the same on all, fetch from first instance */
792+ xcp_ctl = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HYP_XCP_CTL);
793+ 
794+ return REG_GET_FIELD(xcp_ctl, CP_HYP_XCP_CTL, NUM_XCC_IN_XCP);
795+}
621 796 
622static int gfx_v9_4_3_switch_compute_partition(struct amdgpu_device *adev,797static int gfx_v9_4_3_switch_compute_partition(struct amdgpu_device *adev,
623 int num_xccs_per_xcp)798 int num_xccs_per_xcp)
624{799{
625 int ret, i, num_xcc;800 int ret, i, num_xcc;
626- u32 tmp = 0, regval;801+ u32 tmp = 0;
627 802 
628 if (adev->psp.funcs) {803 if (adev->psp.funcs) {
629 ret = psp_spatial_partition(&adev->psp,804 ret = psp_spatial_partition(&adev->psp,
@@ -631,24 +806,23 @@ static int gfx_v9_4_3_switch_compute_partition(struct amdgpu_device *adev,
631 num_xccs_per_xcp);806 num_xccs_per_xcp);
632 if (ret)807 if (ret)
633 return ret;808 return ret;
634- }809+ } else {
810+ num_xcc = NUM_XCC(adev->gfx.xcc_mask);
635 811 
636- num_xcc = NUM_XCC(adev->gfx.xcc_mask);812+ for (i = 0; i < num_xcc; i++) {
637- 813+ tmp = REG_SET_FIELD(tmp, CP_HYP_XCP_CTL, NUM_XCC_IN_XCP,
638- for (i = 0; i < num_xcc; i++) {814+ num_xccs_per_xcp);
639- tmp = REG_SET_FIELD(tmp, CP_HYP_XCP_CTL, NUM_XCC_IN_XCP,815+ tmp = REG_SET_FIELD(tmp, CP_HYP_XCP_CTL, VIRTUAL_XCC_ID,
640- num_xccs_per_xcp);816+ i % num_xccs_per_xcp);
641- tmp = REG_SET_FIELD(tmp, CP_HYP_XCP_CTL, VIRTUAL_XCC_ID,
642- i % num_xccs_per_xcp);
643- regval = RREG32_SOC15(GC, GET_INST(GC, i), regCP_HYP_XCP_CTL);
644- if (regval != tmp)
645 WREG32_SOC15(GC, GET_INST(GC, i), regCP_HYP_XCP_CTL,817 WREG32_SOC15(GC, GET_INST(GC, i), regCP_HYP_XCP_CTL,
646 tmp);818 tmp);
819+ }
820+ ret = 0;
647 }821 }
648 822 
649 adev->gfx.num_xcc_per_xcp = num_xccs_per_xcp;823 adev->gfx.num_xcc_per_xcp = num_xccs_per_xcp;
650 824 
651- return 0;825+ return ret;
652}826}
653 827 
654static int gfx_v9_4_3_ih_to_xcc_inst(struct amdgpu_device *adev, int ih_node)828static int gfx_v9_4_3_ih_to_xcc_inst(struct amdgpu_device *adev, int ih_node)
@@ -673,30 +847,87 @@ static const struct amdgpu_gfx_funcs gfx_v9_4_3_gfx_funcs = {
673 .select_me_pipe_q = &gfx_v9_4_3_select_me_pipe_q,847 .select_me_pipe_q = &gfx_v9_4_3_select_me_pipe_q,
674 .switch_partition_mode = &gfx_v9_4_3_switch_compute_partition,848 .switch_partition_mode = &gfx_v9_4_3_switch_compute_partition,
675 .ih_node_to_logical_xcc = &gfx_v9_4_3_ih_to_xcc_inst,849 .ih_node_to_logical_xcc = &gfx_v9_4_3_ih_to_xcc_inst,
850+ .get_xccs_per_xcp = &gfx_v9_4_3_get_xccs_per_xcp,
851+};
852+ 
853+static int gfx_v9_4_3_aca_bank_parser(struct aca_handle *handle,
854+ struct aca_bank *bank, enum aca_smu_type type,
855+ void *data)
856+{
857+ struct aca_bank_info info;
858+ u64 misc0;
859+ u32 instlo;
860+ int ret;
861+ 
862+ ret = aca_bank_info_decode(bank, &info);
863+ if (ret)
864+ return ret;
865+ 
866+ /* NOTE: overwrite info.die_id with xcd id for gfx */
867+ instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
868+ instlo &= GENMASK(31, 1);
869+ info.die_id = instlo == mmSMNAID_XCD0_MCA_SMU ? 0 : 1;
870+ 
871+ misc0 = bank->regs[ACA_REG_IDX_MISC0];
872+ 
873+ switch (type) {
874+ case ACA_SMU_TYPE_UE:
875+ bank->aca_err_type = ACA_ERROR_TYPE_UE;
876+ ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, 1ULL);
877+ break;
878+ case ACA_SMU_TYPE_CE:
879+ bank->aca_err_type = ACA_ERROR_TYPE_CE;
880+ ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type,
881+ ACA_REG__MISC0__ERRCNT(misc0));
882+ break;
883+ default:
884+ return -EINVAL;
885+ }
886+ 
887+ return ret;
888+}
889+ 
890+static bool gfx_v9_4_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
891+ enum aca_smu_type type, void *data)
892+{
893+ u32 instlo;
894+ 
895+ instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
896+ instlo &= GENMASK(31, 1);
897+ switch (instlo) {
898+ case mmSMNAID_XCD0_MCA_SMU:
899+ case mmSMNAID_XCD1_MCA_SMU:
900+ case mmSMNXCD_XCD0_MCA_SMU:
901+ return true;
902+ default:
903+ break;
904+ }
905+ 
906+ return false;
907+}
908+ 
909+static const struct aca_bank_ops gfx_v9_4_3_aca_bank_ops = {
910+ .aca_bank_parser = gfx_v9_4_3_aca_bank_parser,
911+ .aca_bank_is_valid = gfx_v9_4_3_aca_bank_is_valid,
912+};
913+ 
914+static const struct aca_info gfx_v9_4_3_aca_info = {
915+ .hwip = ACA_HWIP_TYPE_SMU,
916+ .mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK,
917+ .bank_ops = &gfx_v9_4_3_aca_bank_ops,
676};918};
677 919 
678static int gfx_v9_4_3_gpu_early_init(struct amdgpu_device *adev)920static int gfx_v9_4_3_gpu_early_init(struct amdgpu_device *adev)
679{921{
680- u32 gb_addr_config;
681- 
682 adev->gfx.funcs = &gfx_v9_4_3_gfx_funcs;922 adev->gfx.funcs = &gfx_v9_4_3_gfx_funcs;
683 adev->gfx.ras = &gfx_v9_4_3_ras;923 adev->gfx.ras = &gfx_v9_4_3_ras;
684 924 
685- switch (adev->ip_versions[GC_HWIP][0]) {925+ adev->gfx.config.max_hw_contexts = 8;
686- case IP_VERSION(9, 4, 3):926+ adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
687- adev->gfx.config.max_hw_contexts = 8;927+ adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
688- adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;928+ adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
689- adev->gfx.config.sc_prim_fifo_size_backend = 0x100;929+ adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
690- adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;930+ adev->gfx.config.gb_addr_config = GOLDEN_GB_ADDR_CONFIG;
691- adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
692- gb_addr_config = RREG32_SOC15(GC, GET_INST(GC, 0), regGB_ADDR_CONFIG);
693- break;
694- default:
695- BUG();
696- break;
697- }
698- 
699- adev->gfx.config.gb_addr_config = gb_addr_config;
700 931 
701 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<932 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
702 REG_GET_FIELD(933 REG_GET_FIELD(
@@ -775,11 +1006,58 @@ static int gfx_v9_4_3_compute_ring_init(struct amdgpu_device *adev, int ring_id,
775 hw_prio, NULL);1006 hw_prio, NULL);
776}1007}
777 1008 
778-static int gfx_v9_4_3_sw_init(void *handle)1009+static void gfx_v9_4_3_alloc_ip_dump(struct amdgpu_device *adev)
1010+{
1011+ uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9_4_3);
1012+ uint32_t *ptr, num_xcc, inst;
1013+ 
1014+ num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1015+ 
1016+ ptr = kcalloc(reg_count * num_xcc, sizeof(uint32_t), GFP_KERNEL);
1017+ if (!ptr) {
1018+ DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
1019+ adev->gfx.ip_dump_core = NULL;
1020+ } else {
1021+ adev->gfx.ip_dump_core = ptr;
1022+ }
1023+ 
1024+ /* Allocate memory for compute queue registers for all the instances */
1025+ reg_count = ARRAY_SIZE(gc_cp_reg_list_9_4_3);
1026+ inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
1027+ adev->gfx.mec.num_queue_per_pipe;
1028+ 
1029+ ptr = kcalloc(reg_count * inst * num_xcc, sizeof(uint32_t), GFP_KERNEL);
1030+ if (!ptr) {
1031+ DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
1032+ adev->gfx.ip_dump_compute_queues = NULL;
1033+ } else {
1034+ adev->gfx.ip_dump_compute_queues = ptr;
1035+ }
1036+}
1037+ 
1038+static int gfx_v9_4_3_sw_init(struct amdgpu_ip_block *ip_block)
779{1039{
780 int i, j, k, r, ring_id, xcc_id, num_xcc;1040 int i, j, k, r, ring_id, xcc_id, num_xcc;
781- struct amdgpu_kiq *kiq;1041+ struct amdgpu_device *adev = ip_block->adev;
782- struct amdgpu_device *adev = (struct amdgpu_device *)handle;1042+ 
1043+ switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1044+ case IP_VERSION(9, 4, 3):
1045+ case IP_VERSION(9, 4, 4):
1046+ adev->gfx.cleaner_shader_ptr = gfx_9_4_3_cleaner_shader_hex;
1047+ adev->gfx.cleaner_shader_size = sizeof(gfx_9_4_3_cleaner_shader_hex);
1048+ if (adev->gfx.mec_fw_version >= 153) {
1049+ adev->gfx.enable_cleaner_shader = true;
1050+ r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
1051+ if (r) {
1052+ adev->gfx.enable_cleaner_shader = false;
1053+ dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1054+ }
1055+ }
1056+ break;
1057+ default:
1058+ adev->gfx.enable_cleaner_shader = false;
1059+ break;
1060+ }
783 1061 
784 adev->gfx.mec.num_mec = 2;1062 adev->gfx.mec.num_mec = 2;
785 adev->gfx.mec.num_pipe_per_mec = 4;1063 adev->gfx.mec.num_pipe_per_mec = 4;
@@ -792,6 +1070,13 @@ static int gfx_v9_4_3_sw_init(void *handle)
792 if (r)1070 if (r)
793 return r;1071 return r;
794 1072 
1073+ /* Bad opcode Event */
1074+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP,
1075+ GFX_9_0__SRCID__CP_BAD_OPCODE_ERROR,
1076+ &adev->gfx.bad_op_irq);
1077+ if (r)
1078+ return r;
1079+ 
795 /* Privileged reg */1080 /* Privileged reg */
796 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,1081 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
797 &adev->gfx.priv_reg_irq);1082 &adev->gfx.priv_reg_irq);
@@ -847,8 +1132,7 @@ static int gfx_v9_4_3_sw_init(void *handle)
847 return r;1132 return r;
848 }1133 }
849 1134 
850- kiq = &adev->gfx.kiq[xcc_id];1135+ r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
851- r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq, xcc_id);
852 if (r)1136 if (r)
853 return r;1137 return r;
854 1138 
@@ -859,6 +1143,29 @@ static int gfx_v9_4_3_sw_init(void *handle)
859 return r;1143 return r;
860 }1144 }
861 1145 
1146+ adev->gfx.compute_supported_reset =
1147+ amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]);
1148+ switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1149+ case IP_VERSION(9, 4, 3):
1150+ case IP_VERSION(9, 4, 4):
1151+ if ((adev->gfx.mec_fw_version >= 155) &&
1152+ !amdgpu_sriov_vf(adev) &&
1153+ !adev->debug_disable_gpu_ring_reset) {
1154+ adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1155+ adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_PIPE;
1156+ }
1157+ break;
1158+ case IP_VERSION(9, 5, 0):
1159+ if ((adev->gfx.mec_fw_version >= 21) &&
1160+ !amdgpu_sriov_vf(adev) &&
1161+ !adev->debug_disable_gpu_ring_reset) {
1162+ adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1163+ adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_PIPE;
1164+ }
1165+ break;
1166+ default:
1167+ break;
1168+ }
862 r = gfx_v9_4_3_gpu_early_init(adev);1169 r = gfx_v9_4_3_gpu_early_init(adev);
863 if (r)1170 if (r)
864 return r;1171 return r;
@@ -867,17 +1174,19 @@ static int gfx_v9_4_3_sw_init(void *handle)
867 if (r)1174 if (r)
868 return r;1175 return r;
869 1176 
1177+ r = amdgpu_gfx_sysfs_init(adev);
1178+ if (r)
1179+ return r;
870 1180 
871- if (!amdgpu_sriov_vf(adev))1181+ gfx_v9_4_3_alloc_ip_dump(adev);
872- r = amdgpu_gfx_sysfs_init(adev);
873 1182 
874- return r;1183+ return 0;
875}1184}
876 1185 
877-static int gfx_v9_4_3_sw_fini(void *handle)1186+static int gfx_v9_4_3_sw_fini(struct amdgpu_ip_block *ip_block)
878{1187{
879 int i, num_xcc;1188 int i, num_xcc;
880- struct amdgpu_device *adev = (struct amdgpu_device *)handle;1189+ struct amdgpu_device *adev = ip_block->adev;
881 1190 
882 num_xcc = NUM_XCC(adev->gfx.xcc_mask);1191 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
883 for (i = 0; i < adev->gfx.num_compute_rings * num_xcc; i++)1192 for (i = 0; i < adev->gfx.num_compute_rings * num_xcc; i++)
@@ -889,11 +1198,15 @@ static int gfx_v9_4_3_sw_fini(void *handle)
889 amdgpu_gfx_kiq_fini(adev, i);1198 amdgpu_gfx_kiq_fini(adev, i);
890 }1199 }
891 1200 
1201+ amdgpu_gfx_cleaner_shader_sw_fini(adev);
1202+ 
892 gfx_v9_4_3_mec_fini(adev);1203 gfx_v9_4_3_mec_fini(adev);
893 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);1204 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
894 gfx_v9_4_3_free_microcode(adev);1205 gfx_v9_4_3_free_microcode(adev);
895- if (!amdgpu_sriov_vf(adev))1206+ amdgpu_gfx_sysfs_fini(adev);
896- amdgpu_gfx_sysfs_fini(adev);1207+ 
1208+ kfree(adev->gfx.ip_dump_core);
1209+ kfree(adev->gfx.ip_dump_compute_queues);
897 1210 
898 return 0;1211 return 0;
899}1212}
@@ -934,8 +1247,10 @@ static void gfx_v9_4_3_xcc_init_compute_vmid(struct amdgpu_device *adev,
934 soc15_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));1247 soc15_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
935 mutex_unlock(&adev->srbm_mutex);1248 mutex_unlock(&adev->srbm_mutex);
936 1249 
937- /* Initialize all compute VMIDs to have no GDS, GWS, or OA1250+ /*
938- acccess. These should be enabled by FW for target VMIDs. */1251+ * Initialize all compute VMIDs to have no GDS, GWS, or OA
1252+ * access. These should be enabled by FW for target VMIDs.
1253+ */
939 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {1254 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
940 WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_BASE, 2 * i, 0);1255 WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_BASE, 2 * i, 0);
941 WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_SIZE, 2 * i, 0);1256 WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_SIZE, 2 * i, 0);
@@ -962,6 +1277,22 @@ static void gfx_v9_4_3_xcc_init_gds_vmid(struct amdgpu_device *adev, int xcc_id)
962 }1277 }
963}1278}
964 1279 
1280+/* For ASICs that needs xnack chain and MEC version supports, set SG_CONFIG1
1281+ * DISABLE_XNACK_CHECK_IN_RETRY_DISABLE bit and inform KFD to set xnack_chain
1282+ * bit in SET_RESOURCES
1283+ */
1284+static void gfx_v9_4_3_xcc_init_sq(struct amdgpu_device *adev, int xcc_id)
1285+{
1286+ uint32_t data;
1287+ 
1288+ if (!(adev->gmc.xnack_flags & AMDGPU_GMC_XNACK_FLAG_CHAIN))
1289+ return;
1290+ 
1291+ data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_CONFIG1);
1292+ data = REG_SET_FIELD(data, SQ_CONFIG1, DISABLE_XNACK_CHECK_IN_RETRY_DISABLE, 1);
1293+ WREG32_SOC15(GC, xcc_id, regSQ_CONFIG1, data);
1294+}
1295+ 
965static void gfx_v9_4_3_xcc_constants_init(struct amdgpu_device *adev,1296static void gfx_v9_4_3_xcc_constants_init(struct amdgpu_device *adev,
966 int xcc_id)1297 int xcc_id)
967{1298{
@@ -1006,6 +1337,7 @@ static void gfx_v9_4_3_xcc_constants_init(struct amdgpu_device *adev,
1006 1337 
1007 gfx_v9_4_3_xcc_init_compute_vmid(adev, xcc_id);1338 gfx_v9_4_3_xcc_init_compute_vmid(adev, xcc_id);
1008 gfx_v9_4_3_xcc_init_gds_vmid(adev, xcc_id);1339 gfx_v9_4_3_xcc_init_gds_vmid(adev, xcc_id);
1340+ gfx_v9_4_3_xcc_init_sq(adev, xcc_id);
1009}1341}
1010 1342 
1011static void gfx_v9_4_3_constants_init(struct amdgpu_device *adev)1343static void gfx_v9_4_3_constants_init(struct amdgpu_device *adev)
@@ -1018,6 +1350,22 @@ static void gfx_v9_4_3_constants_init(struct amdgpu_device *adev)
1018 adev->gfx.config.db_debug2 =1350 adev->gfx.config.db_debug2 =
1019 RREG32_SOC15(GC, GET_INST(GC, 0), regDB_DEBUG2);1351 RREG32_SOC15(GC, GET_INST(GC, 0), regDB_DEBUG2);
1020 1352 
1353+ switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1354+ /* ToDo: GC 9.4.4 */
1355+ case IP_VERSION(9, 4, 3):
1356+ if (adev->gfx.mec_fw_version >= 184 &&
1357+ (amdgpu_sriov_reg_access_sq_config(adev) ||
1358+ !amdgpu_sriov_vf(adev)))
1359+ adev->gmc.xnack_flags |= AMDGPU_GMC_XNACK_FLAG_CHAIN;
1360+ break;
1361+ case IP_VERSION(9, 5, 0):
1362+ if (adev->gfx.mec_fw_version >= 23)
1363+ adev->gmc.xnack_flags |= AMDGPU_GMC_XNACK_FLAG_CHAIN;
1364+ break;
1365+ default:
1366+ break;
1367+ }
1368+ 
1021 for (i = 0; i < num_xcc; i++)1369 for (i = 0; i < num_xcc; i++)
1022 gfx_v9_4_3_xcc_constants_init(adev, i);1370 gfx_v9_4_3_xcc_constants_init(adev, i);
1023}1371}
@@ -1033,10 +1381,8 @@ static void gfx_v9_4_3_xcc_init_pg(struct amdgpu_device *adev, int xcc_id)
1033{1381{
1034 /*1382 /*
1035 * Rlc save restore list is workable since v2_1.1383 * Rlc save restore list is workable since v2_1.
1036- * And it's needed by gfxoff feature.
1037 */1384 */
1038- if (adev->gfx.rlc.is_rlc_v2_1)1385+ gfx_v9_4_3_xcc_enable_save_restore_machine(adev, xcc_id);
1039- gfx_v9_4_3_xcc_enable_save_restore_machine(adev, xcc_id);
1040}1386}
1041 1387 
1042static void gfx_v9_4_3_xcc_disable_gpa_mode(struct amdgpu_device *adev, int xcc_id)1388static void gfx_v9_4_3_xcc_disable_gpa_mode(struct amdgpu_device *adev, int xcc_id)
@@ -1102,13 +1448,14 @@ static void gfx_v9_4_3_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
1102 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX);1448 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX);
1103 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPARE_INT);1449 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPARE_INT);
1104 }1450 }
1451+ adev->gfx.rlc.rlcg_reg_access_supported = true;
1105}1452}
1106 1453 
1107static int gfx_v9_4_3_rlc_init(struct amdgpu_device *adev)1454static int gfx_v9_4_3_rlc_init(struct amdgpu_device *adev)
1108{1455{
1109 /* init spm vmid with 0xf */1456 /* init spm vmid with 0xf */
1110 if (adev->gfx.rlc.funcs->update_spm_vmid)1457 if (adev->gfx.rlc.funcs->update_spm_vmid)
1111- adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);1458+ adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
1112 1459 
1113 return 0;1460 return 0;
1114}1461}
@@ -1319,24 +1666,26 @@ static int gfx_v9_4_3_rlc_resume(struct amdgpu_device *adev)
1319 return 0;1666 return 0;
1320}1667}
1321 1668 
1322-static void gfx_v9_4_3_update_spm_vmid(struct amdgpu_device *adev,1669+static void gfx_v9_4_3_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring,
1323 unsigned vmid)1670 unsigned vmid)
1324{1671{
1325- u32 reg, data;1672+ u32 reg, pre_data, data;
1326 1673 
1327 reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regRLC_SPM_MC_CNTL);1674 reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regRLC_SPM_MC_CNTL);
1328- if (amdgpu_sriov_is_pp_one_vf(adev))1675+ if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev))
1329- data = RREG32_NO_KIQ(reg);1676+ pre_data = RREG32_NO_KIQ(reg);
1330 else1677 else
1331- data = RREG32(reg);1678+ pre_data = RREG32(reg);
1332 1679 
1333- data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;1680+ data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK);
1334 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;1681 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
1335 1682 
1336- if (amdgpu_sriov_is_pp_one_vf(adev))1683+ if (pre_data != data) {
1337- WREG32_SOC15_NO_KIQ(GC, GET_INST(GC, 0), regRLC_SPM_MC_CNTL, data);1684+ if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) {
1338- else1685+ WREG32_SOC15_NO_KIQ(GC, GET_INST(GC, 0), regRLC_SPM_MC_CNTL, data);
1339- WREG32_SOC15(GC, GET_INST(GC, 0), regRLC_SPM_MC_CNTL, data);1686+ } else
1687+ WREG32_SOC15(GC, GET_INST(GC, 0), regRLC_SPM_MC_CNTL, data);
1688+ }
1340}1689}
1341 1690 
1342static const struct soc15_reg_rlcg rlcg_access_gc_9_4_3[] = {1691static const struct soc15_reg_rlcg rlcg_access_gc_9_4_3[] = {
@@ -1385,7 +1734,15 @@ static void gfx_v9_4_3_xcc_cp_compute_enable(struct amdgpu_device *adev,
1385 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MEC_CNTL, 0);1734 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MEC_CNTL, 0);
1386 } else {1735 } else {
1387 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MEC_CNTL,1736 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MEC_CNTL,
1388- (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));1737+ (CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK |
1738+ CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK |
1739+ CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK |
1740+ CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK |
1741+ CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK |
1742+ CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK |
1743+ CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK |
1744+ CP_MEC_CNTL__MEC_ME1_HALT_MASK |
1745+ CP_MEC_CNTL__MEC_ME2_HALT_MASK));
1389 adev->gfx.kiq[xcc_id].ring.sched.ready = false;1746 adev->gfx.kiq[xcc_id].ring.sched.ready = false;
1390 }1747 }
1391 udelay(50);1748 udelay(50);
@@ -1449,9 +1806,7 @@ static void gfx_v9_4_3_xcc_kiq_setting(struct amdgpu_ring *ring, int xcc_id)
1449 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS);1806 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS);
1450 tmp &= 0xffffff00;1807 tmp &= 0xffffff00;
1451 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);1808 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1452- WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);1809+ WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp | 0x80);
1453- tmp |= 0x80;
1454- WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
1455}1810}
1456 1811 
1457static void gfx_v9_4_3_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)1812static void gfx_v9_4_3_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
@@ -1512,6 +1867,9 @@ static int gfx_v9_4_3_xcc_mqd_init(struct amdgpu_ring *ring, int xcc_id)
1512 DOORBELL_SOURCE, 0);1867 DOORBELL_SOURCE, 0);
1513 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,1868 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1514 DOORBELL_HIT, 0);1869 DOORBELL_HIT, 0);
1870+ if (amdgpu_sriov_multi_vf_mode(adev))
1871+ tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1872+ DOORBELL_MODE, 1);
1515 } else {1873 } else {
1516 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,1874 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1517 DOORBELL_EN, 0);1875 DOORBELL_EN, 0);
@@ -1796,7 +2154,8 @@ static int gfx_v9_4_3_xcc_kiq_init_queue(struct amdgpu_ring *ring, int xcc_id)
1796 return 0;2154 return 0;
1797}2155}
1798 2156 
1799-static int gfx_v9_4_3_xcc_kcq_init_queue(struct amdgpu_ring *ring, int xcc_id)2157+static void gfx_v9_4_3_xcc_kcq_init_queue(struct amdgpu_ring *ring, int xcc_id,
2158+ bool restore)
1800{2159{
1801 struct amdgpu_device *adev = ring->adev;2160 struct amdgpu_device *adev = ring->adev;
1802 struct v9_mqd *mqd = ring->mqd_ptr;2161 struct v9_mqd *mqd = ring->mqd_ptr;
@@ -1808,8 +2167,8 @@ static int gfx_v9_4_3_xcc_kcq_init_queue(struct amdgpu_ring *ring, int xcc_id)
1808 */2167 */
1809 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];2168 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
1810 2169 
1811- if (!tmp_mqd->cp_hqd_pq_control ||2170+ if (!restore && (!tmp_mqd->cp_hqd_pq_control ||
1812- (!amdgpu_in_reset(adev) && !adev->in_suspend)) {2171+ (!amdgpu_in_reset(adev) && !adev->in_suspend))) {
1813 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));2172 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
1814 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;2173 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
1815 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;2174 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
@@ -1830,8 +2189,6 @@ static int gfx_v9_4_3_xcc_kcq_init_queue(struct amdgpu_ring *ring, int xcc_id)
1830 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);2189 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);
1831 amdgpu_ring_clear_ring(ring);2190 amdgpu_ring_clear_ring(ring);
1832 }2191 }
1833- 
1834- return 0;
1835}2192}
1836 2193 
1837static int gfx_v9_4_3_xcc_kcq_fini_register(struct amdgpu_device *adev, int xcc_id)2194static int gfx_v9_4_3_xcc_kcq_fini_register(struct amdgpu_device *adev, int xcc_id)
@@ -1857,55 +2214,25 @@ static int gfx_v9_4_3_xcc_kcq_fini_register(struct amdgpu_device *adev, int xcc_
1857 2214 
1858static int gfx_v9_4_3_xcc_kiq_resume(struct amdgpu_device *adev, int xcc_id)2215static int gfx_v9_4_3_xcc_kiq_resume(struct amdgpu_device *adev, int xcc_id)
1859{2216{
1860- struct amdgpu_ring *ring;2217+ gfx_v9_4_3_xcc_kiq_init_queue(&adev->gfx.kiq[xcc_id].ring, xcc_id);
1861- int r;
1862- 
1863- ring = &adev->gfx.kiq[xcc_id].ring;
1864- 
1865- r = amdgpu_bo_reserve(ring->mqd_obj, false);
1866- if (unlikely(r != 0))
1867- return r;
1868- 
1869- r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
1870- if (unlikely(r != 0)) {
1871- amdgpu_bo_unreserve(ring->mqd_obj);
1872- return r;
1873- }
1874- 
1875- gfx_v9_4_3_xcc_kiq_init_queue(ring, xcc_id);
1876- amdgpu_bo_kunmap(ring->mqd_obj);
1877- ring->mqd_ptr = NULL;
1878- amdgpu_bo_unreserve(ring->mqd_obj);
1879 return 0;2218 return 0;
1880}2219}
1881 2220 
1882static int gfx_v9_4_3_xcc_kcq_resume(struct amdgpu_device *adev, int xcc_id)2221static int gfx_v9_4_3_xcc_kcq_resume(struct amdgpu_device *adev, int xcc_id)
1883{2222{
1884- struct amdgpu_ring *ring = NULL;2223+ struct amdgpu_ring *ring;
1885- int r = 0, i;2224+ int i;
1886 2225 
1887 gfx_v9_4_3_xcc_cp_compute_enable(adev, true, xcc_id);2226 gfx_v9_4_3_xcc_cp_compute_enable(adev, true, xcc_id);
1888 2227 
1889 for (i = 0; i < adev->gfx.num_compute_rings; i++) {2228 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
1890- ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings];2229+ ring = &adev->gfx.compute_ring[i + xcc_id *
2230+ adev->gfx.num_compute_rings];
1891 2231 
1892- r = amdgpu_bo_reserve(ring->mqd_obj, false);2232+ gfx_v9_4_3_xcc_kcq_init_queue(ring, xcc_id, false);
1893- if (unlikely(r != 0))
1894- goto done;
1895- r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
1896- if (!r) {
1897- r = gfx_v9_4_3_xcc_kcq_init_queue(ring, xcc_id);
1898- amdgpu_bo_kunmap(ring->mqd_obj);
1899- ring->mqd_ptr = NULL;
1900- }
1901- amdgpu_bo_unreserve(ring->mqd_obj);
1902- if (r)
1903- goto done;
1904 }2233 }
1905 2234 
1906- r = amdgpu_gfx_enable_kcq(adev, xcc_id);2235+ return amdgpu_gfx_enable_kcq(adev, xcc_id);
1907-done:
1908- return r;
1909}2236}
1910 2237 
1911static int gfx_v9_4_3_xcc_cp_resume(struct amdgpu_device *adev, int xcc_id)2238static int gfx_v9_4_3_xcc_cp_resume(struct amdgpu_device *adev, int xcc_id)
@@ -1921,6 +2248,8 @@ static int gfx_v9_4_3_xcc_cp_resume(struct amdgpu_device *adev, int xcc_id)
1921 r = gfx_v9_4_3_xcc_cp_compute_load_microcode(adev, xcc_id);2248 r = gfx_v9_4_3_xcc_cp_compute_load_microcode(adev, xcc_id);
1922 if (r)2249 if (r)
1923 return r;2250 return r;
2251+ } else {
2252+ gfx_v9_4_3_xcc_cp_compute_enable(adev, false, xcc_id);
1924 }2253 }
1925 2254 
1926 r = gfx_v9_4_3_xcc_kiq_resume(adev, xcc_id);2255 r = gfx_v9_4_3_xcc_kiq_resume(adev, xcc_id);
@@ -1946,18 +2275,33 @@ static int gfx_v9_4_3_xcc_cp_resume(struct amdgpu_device *adev, int xcc_id)
1946 2275 
1947static int gfx_v9_4_3_cp_resume(struct amdgpu_device *adev)2276static int gfx_v9_4_3_cp_resume(struct amdgpu_device *adev)
1948{2277{
1949- int r = 0, i, num_xcc;2278+ int r = 0, i, num_xcc, num_xcp, num_xcc_per_xcp;
1950 2279 
1951- if (amdgpu_xcp_query_partition_mode(adev->xcp_mgr,2280+ num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1952- AMDGPU_XCP_FL_NONE) ==2281+ if (amdgpu_sriov_vf(adev)) {
1953- AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE)2282+ enum amdgpu_gfx_partition mode;
1954- r = amdgpu_xcp_switch_partition_mode(adev->xcp_mgr,
1955- amdgpu_user_partt_mode);
1956 2283 
2284+ mode = amdgpu_xcp_query_partition_mode(adev->xcp_mgr,
2285+ AMDGPU_XCP_FL_NONE);
2286+ if (mode == AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE)
2287+ return -EINVAL;
2288+ num_xcc_per_xcp = gfx_v9_4_3_get_xccs_per_xcp(adev);
2289+ adev->gfx.num_xcc_per_xcp = num_xcc_per_xcp;
2290+ num_xcp = num_xcc / num_xcc_per_xcp;
2291+ r = amdgpu_xcp_init(adev->xcp_mgr, num_xcp, mode);
2292+ 
2293+ } else {
2294+ if (adev->in_suspend)
2295+ amdgpu_xcp_restore_partition_mode(adev->xcp_mgr);
2296+ else if (amdgpu_xcp_query_partition_mode(adev->xcp_mgr,
2297+ AMDGPU_XCP_FL_NONE) ==
2298+ AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE)
2299+ r = amdgpu_xcp_switch_partition_mode(
2300+ adev->xcp_mgr, amdgpu_user_partt_mode);
2301+ }
1957 if (r)2302 if (r)
1958 return r;2303 return r;
1959 2304 
1960- num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1961 for (i = 0; i < num_xcc; i++) {2305 for (i = 0; i < num_xcc; i++) {
1962 r = gfx_v9_4_3_xcc_cp_resume(adev, i);2306 r = gfx_v9_4_3_xcc_cp_resume(adev, i);
1963 if (r)2307 if (r)
@@ -1967,12 +2311,6 @@ static int gfx_v9_4_3_cp_resume(struct amdgpu_device *adev)
1967 return 0;2311 return 0;
1968}2312}
1969 2313 
1970-static void gfx_v9_4_3_xcc_cp_enable(struct amdgpu_device *adev, bool enable,
1971- int xcc_id)
1972-{
1973- gfx_v9_4_3_xcc_cp_compute_enable(adev, enable, xcc_id);
1974-}
1975- 
1976static void gfx_v9_4_3_xcc_fini(struct amdgpu_device *adev, int xcc_id)2314static void gfx_v9_4_3_xcc_fini(struct amdgpu_device *adev, int xcc_id)
1977{2315{
1978 if (amdgpu_gfx_disable_kcq(adev, xcc_id))2316 if (amdgpu_gfx_disable_kcq(adev, xcc_id))
@@ -2004,13 +2342,16 @@ static void gfx_v9_4_3_xcc_fini(struct amdgpu_device *adev, int xcc_id)
2004 }2342 }
2005 2343 
2006 gfx_v9_4_3_xcc_kcq_fini_register(adev, xcc_id);2344 gfx_v9_4_3_xcc_kcq_fini_register(adev, xcc_id);
2007- gfx_v9_4_3_xcc_cp_enable(adev, false, xcc_id);2345+ gfx_v9_4_3_xcc_cp_compute_enable(adev, false, xcc_id);
2008}2346}
2009 2347 
2010-static int gfx_v9_4_3_hw_init(void *handle)2348+static int gfx_v9_4_3_hw_init(struct amdgpu_ip_block *ip_block)
2011{2349{
2012 int r;2350 int r;
2013- struct amdgpu_device *adev = (struct amdgpu_device *)handle;2351+ struct amdgpu_device *adev = ip_block->adev;
2352+ 
2353+ amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size,
2354+ adev->gfx.cleaner_shader_ptr);
2014 2355 
2015 if (!amdgpu_sriov_vf(adev))2356 if (!amdgpu_sriov_vf(adev))
2016 gfx_v9_4_3_init_golden_registers(adev);2357 gfx_v9_4_3_init_golden_registers(adev);
@@ -2028,13 +2369,14 @@ static int gfx_v9_4_3_hw_init(void *handle)
2028 return r;2369 return r;
2029}2370}
2030 2371 
2031-static int gfx_v9_4_3_hw_fini(void *handle)2372+static int gfx_v9_4_3_hw_fini(struct amdgpu_ip_block *ip_block)
2032{2373{
2033- struct amdgpu_device *adev = (struct amdgpu_device *)handle;2374+ struct amdgpu_device *adev = ip_block->adev;
2034 int i, num_xcc;2375 int i, num_xcc;
2035 2376 
2036 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);2377 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
2037 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);2378 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
2379+ amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
2038 2380 
2039 num_xcc = NUM_XCC(adev->gfx.xcc_mask);2381 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2040 for (i = 0; i < num_xcc; i++) {2382 for (i = 0; i < num_xcc; i++) {
@@ -2044,19 +2386,19 @@ static int gfx_v9_4_3_hw_fini(void *handle)
2044 return 0;2386 return 0;
2045}2387}
2046 2388 
2047-static int gfx_v9_4_3_suspend(void *handle)2389+static int gfx_v9_4_3_suspend(struct amdgpu_ip_block *ip_block)
2048{2390{
2049- return gfx_v9_4_3_hw_fini(handle);2391+ return gfx_v9_4_3_hw_fini(ip_block);
2050}2392}
2051 2393 
2052-static int gfx_v9_4_3_resume(void *handle)2394+static int gfx_v9_4_3_resume(struct amdgpu_ip_block *ip_block)
2053{2395{
2054- return gfx_v9_4_3_hw_init(handle);2396+ return gfx_v9_4_3_hw_init(ip_block);
2055}2397}
2056 2398 
2057-static bool gfx_v9_4_3_is_idle(void *handle)2399+static bool gfx_v9_4_3_is_idle(struct amdgpu_ip_block *ip_block)
2058{2400{
2059- struct amdgpu_device *adev = (struct amdgpu_device *)handle;2401+ struct amdgpu_device *adev = ip_block->adev;
2060 int i, num_xcc;2402 int i, num_xcc;
2061 2403 
2062 num_xcc = NUM_XCC(adev->gfx.xcc_mask);2404 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
@@ -2068,24 +2410,24 @@ static bool gfx_v9_4_3_is_idle(void *handle)
2068 return true;2410 return true;
2069}2411}
2070 2412 
2071-static int gfx_v9_4_3_wait_for_idle(void *handle)2413+static int gfx_v9_4_3_wait_for_idle(struct amdgpu_ip_block *ip_block)
2072{2414{
2073 unsigned i;2415 unsigned i;
2074- struct amdgpu_device *adev = (struct amdgpu_device *)handle;2416+ struct amdgpu_device *adev = ip_block->adev;
2075 2417 
2076 for (i = 0; i < adev->usec_timeout; i++) {2418 for (i = 0; i < adev->usec_timeout; i++) {
2077- if (gfx_v9_4_3_is_idle(handle))2419+ if (gfx_v9_4_3_is_idle(ip_block))
2078 return 0;2420 return 0;
2079 udelay(1);2421 udelay(1);
2080 }2422 }
2081 return -ETIMEDOUT;2423 return -ETIMEDOUT;
2082}2424}
2083 2425 
2084-static int gfx_v9_4_3_soft_reset(void *handle)2426+static int gfx_v9_4_3_soft_reset(struct amdgpu_ip_block *ip_block)
2085{2427{
2086 u32 grbm_soft_reset = 0;2428 u32 grbm_soft_reset = 0;
2087 u32 tmp;2429 u32 tmp;
2088- struct amdgpu_device *adev = (struct amdgpu_device *)handle;2430+ struct amdgpu_device *adev = ip_block->adev;
2089 2431 
2090 /* GRBM_STATUS */2432 /* GRBM_STATUS */
2091 tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_STATUS);2433 tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_STATUS);
@@ -2120,19 +2462,17 @@ static int gfx_v9_4_3_soft_reset(void *handle)
2120 /* Disable MEC parsing/prefetching */2462 /* Disable MEC parsing/prefetching */
2121 gfx_v9_4_3_xcc_cp_compute_enable(adev, false, 0);2463 gfx_v9_4_3_xcc_cp_compute_enable(adev, false, 0);
2122 2464 
2123- if (grbm_soft_reset) {2465+ tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET);
2124- tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET);2466+ tmp |= grbm_soft_reset;
2125- tmp |= grbm_soft_reset;2467+ dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
2126- dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);2468+ WREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET, tmp);
2127- WREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET, tmp);2469+ tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET);
2128- tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET);
2129 2470 
2130- udelay(50);2471+ udelay(50);
2131 2472 
2132- tmp &= ~grbm_soft_reset;2473+ tmp &= ~grbm_soft_reset;
2133- WREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET, tmp);2474+ WREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET, tmp);
2134- tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET);2475+ tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET);
2135- }
2136 2476 
2137 /* Wait a little for things to settle down */2477 /* Wait a little for things to settle down */
2138 udelay(50);2478 udelay(50);
@@ -2169,9 +2509,9 @@ static void gfx_v9_4_3_ring_emit_gds_switch(struct amdgpu_ring *ring,
2169 (1 << (oa_size + oa_base)) - (1 << oa_base));2509 (1 << (oa_size + oa_base)) - (1 << oa_base));
2170}2510}
2171 2511 
2172-static int gfx_v9_4_3_early_init(void *handle)2512+static int gfx_v9_4_3_early_init(struct amdgpu_ip_block *ip_block)
2173{2513{
2174- struct amdgpu_device *adev = (struct amdgpu_device *)handle;2514+ struct amdgpu_device *adev = ip_block->adev;
2175 2515 
2176 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),2516 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
2177 AMDGPU_MAX_COMPUTE_RINGS);2517 AMDGPU_MAX_COMPUTE_RINGS);
@@ -2187,9 +2527,9 @@ static int gfx_v9_4_3_early_init(void *handle)
2187 return gfx_v9_4_3_init_microcode(adev);2527 return gfx_v9_4_3_init_microcode(adev);
2188}2528}
2189 2529 
2190-static int gfx_v9_4_3_late_init(void *handle)2530+static int gfx_v9_4_3_late_init(struct amdgpu_ip_block *ip_block)
2191{2531{
2192- struct amdgpu_device *adev = (struct amdgpu_device *)handle;2532+ struct amdgpu_device *adev = ip_block->adev;
2193 int r;2533 int r;
2194 2534 
2195 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);2535 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
@@ -2200,6 +2540,10 @@ static int gfx_v9_4_3_late_init(void *handle)
2200 if (r)2540 if (r)
2201 return r;2541 return r;
2202 2542 
2543+ r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
2544+ if (r)
2545+ return r;
2546+ 
2203 if (adev->gfx.ras &&2547 if (adev->gfx.ras &&
2204 adev->gfx.ras->enable_watchdog_timer)2548 adev->gfx.ras->enable_watchdog_timer)
2205 adev->gfx.ras->enable_watchdog_timer(adev);2549 adev->gfx.ras->enable_watchdog_timer(adev);
@@ -2334,10 +2678,10 @@ gfx_v9_4_3_xcc_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
2334 if (def != data)2678 if (def != data)
2335 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);2679 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
2336 2680 
2337- /* enable cgcg FSM(0x0000363F) */2681+ /* CGCG Hysteresis: 400us */
2338 def = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL);2682 def = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL);
2339 2683 
2340- data = (0x362684+ data = (0x2710
2341 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |2685 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
2342 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;2686 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
2343 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)2687 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
@@ -2346,10 +2690,10 @@ gfx_v9_4_3_xcc_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
2346 if (def != data)2690 if (def != data)
2347 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, data);2691 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, data);
2348 2692 
2349- /* set IDLE_POLL_COUNT(0x00900100) */2693+ /* set IDLE_POLL_COUNT(0x33450100)*/
2350 def = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL);2694 def = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL);
2351 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |2695 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
2352- (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);2696+ (0x3345 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2353 if (def != data)2697 if (def != data)
2354 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL, data);2698 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL, data);
2355 } else {2699 } else {
@@ -2414,37 +2758,32 @@ static const struct amdgpu_rlc_funcs gfx_v9_4_3_rlc_funcs = {
2414 .is_rlcg_access_range = gfx_v9_4_3_is_rlcg_access_range,2758 .is_rlcg_access_range = gfx_v9_4_3_is_rlcg_access_range,
2415};2759};
2416 2760 
2417-static int gfx_v9_4_3_set_powergating_state(void *handle,2761+static int gfx_v9_4_3_set_powergating_state(struct amdgpu_ip_block *ip_block,
2418 enum amd_powergating_state state)2762 enum amd_powergating_state state)
2419{2763{
2420 return 0;2764 return 0;
2421}2765}
2422 2766 
2423-static int gfx_v9_4_3_set_clockgating_state(void *handle,2767+static int gfx_v9_4_3_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2424 enum amd_clockgating_state state)2768 enum amd_clockgating_state state)
2425{2769{
2426- struct amdgpu_device *adev = (struct amdgpu_device *)handle;2770+ struct amdgpu_device *adev = ip_block->adev;
2427 int i, num_xcc;2771 int i, num_xcc;
2428 2772 
2429 if (amdgpu_sriov_vf(adev))2773 if (amdgpu_sriov_vf(adev))
2430 return 0;2774 return 0;
2431 2775 
2432 num_xcc = NUM_XCC(adev->gfx.xcc_mask);2776 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2433- switch (adev->ip_versions[GC_HWIP][0]) {2777+ for (i = 0; i < num_xcc; i++)
2434- case IP_VERSION(9, 4, 3):2778+ gfx_v9_4_3_xcc_update_gfx_clock_gating(
2435- for (i = 0; i < num_xcc; i++)2779+ adev, state == AMD_CG_STATE_GATE, i);
2436- gfx_v9_4_3_xcc_update_gfx_clock_gating(2780+ 
2437- adev, state == AMD_CG_STATE_GATE, i);
2438- break;
2439- default:
2440- break;
2441- }
2442 return 0;2781 return 0;
2443}2782}
2444 2783 
2445-static void gfx_v9_4_3_get_clockgating_state(void *handle, u64 *flags)2784+static void gfx_v9_4_3_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
2446{2785{
2447- struct amdgpu_device *adev = (struct amdgpu_device *)handle;2786+ struct amdgpu_device *adev = ip_block->adev;
2448 int data;2787 int data;
2449 2788 
2450 if (amdgpu_sriov_vf(adev))2789 if (amdgpu_sriov_vf(adev))
@@ -2652,6 +2991,8 @@ static void gfx_v9_4_3_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
2652{2991{
2653 struct amdgpu_device *adev = ring->adev;2992 struct amdgpu_device *adev = ring->adev;
2654 2993 
2994+ reg = gfx_v9_4_3_normalize_xcc_reg_offset(reg);
2995+ 
2655 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));2996 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
2656 amdgpu_ring_write(ring, 0 | /* src: register*/2997 amdgpu_ring_write(ring, 0 | /* src: register*/
2657 (5 << 8) | /* dst: memory */2998 (5 << 8) | /* dst: memory */
@@ -2669,6 +3010,8 @@ static void gfx_v9_4_3_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
2669{3010{
2670 uint32_t cmd = 0;3011 uint32_t cmd = 0;
2671 3012 
3013+ reg = gfx_v9_4_3_normalize_xcc_reg_offset(reg);
3014+ 
2672 switch (ring->funcs->type) {3015 switch (ring->funcs->type) {
2673 case AMDGPU_RING_TYPE_GFX:3016 case AMDGPU_RING_TYPE_GFX:
2674 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;3017 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
@@ -2701,6 +3044,21 @@ static void gfx_v9_4_3_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
2701 ref, mask);3044 ref, mask);
2702}3045}
2703 3046 
3047+static void gfx_v9_4_3_ring_soft_recovery(struct amdgpu_ring *ring,
3048+ unsigned vmid)
3049+{
3050+ struct amdgpu_device *adev = ring->adev;
3051+ uint32_t value = 0;
3052+ 
3053+ value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
3054+ value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
3055+ value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
3056+ value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
3057+ amdgpu_gfx_rlc_enter_safe_mode(adev, ring->xcc_id);
3058+ WREG32_SOC15(GC, GET_INST(GC, ring->xcc_id), regSQ_CMD, value);
3059+ amdgpu_gfx_rlc_exit_safe_mode(adev, ring->xcc_id);
3060+}
3061+ 
2704static void gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(3062static void gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
2705 struct amdgpu_device *adev, int me, int pipe,3063 struct amdgpu_device *adev, int me, int pipe,
2706 enum amdgpu_interrupt_state state, int xcc_id)3064 enum amdgpu_interrupt_state state, int xcc_id)
@@ -2738,37 +3096,119 @@ static void gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
2738 3096 
2739 switch (state) {3097 switch (state) {
2740 case AMDGPU_IRQ_STATE_DISABLE:3098 case AMDGPU_IRQ_STATE_DISABLE:
2741- mec_int_cntl = RREG32(mec_int_cntl_reg);3099+ mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id);
2742 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,3100 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
2743 TIME_STAMP_INT_ENABLE, 0);3101 TIME_STAMP_INT_ENABLE, 0);
2744- WREG32(mec_int_cntl_reg, mec_int_cntl);3102+ WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id);
2745 break;3103 break;
2746 case AMDGPU_IRQ_STATE_ENABLE:3104 case AMDGPU_IRQ_STATE_ENABLE:
2747- mec_int_cntl = RREG32(mec_int_cntl_reg);3105+ mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id);
2748 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,3106 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
2749 TIME_STAMP_INT_ENABLE, 1);3107 TIME_STAMP_INT_ENABLE, 1);
2750- WREG32(mec_int_cntl_reg, mec_int_cntl);3108+ WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id);
2751 break;3109 break;
2752 default:3110 default:
2753 break;3111 break;
2754 }3112 }
2755}3113}
2756 3114 
3115+static u32 gfx_v9_4_3_get_cpc_int_cntl(struct amdgpu_device *adev,
3116+ int xcc_id, int me, int pipe)
3117+{
3118+ /*
3119+ * amdgpu controls only the first MEC. That's why this function only
3120+ * handles the setting of interrupts for this specific MEC. All other
3121+ * pipes' interrupts are set by amdkfd.
3122+ */
3123+ if (me != 1)
3124+ return 0;
3125+ 
3126+ switch (pipe) {
3127+ case 0:
3128+ return SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE0_INT_CNTL);
3129+ case 1:
3130+ return SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE1_INT_CNTL);
3131+ case 2:
3132+ return SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE2_INT_CNTL);
3133+ case 3:
3134+ return SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE3_INT_CNTL);
3135+ default:
3136+ return 0;
3137+ }
3138+}
3139+ 
2757static int gfx_v9_4_3_set_priv_reg_fault_state(struct amdgpu_device *adev,3140static int gfx_v9_4_3_set_priv_reg_fault_state(struct amdgpu_device *adev,
2758 struct amdgpu_irq_src *source,3141 struct amdgpu_irq_src *source,
2759 unsigned type,3142 unsigned type,
2760 enum amdgpu_interrupt_state state)3143 enum amdgpu_interrupt_state state)
2761{3144{
2762- int i, num_xcc;3145+ u32 mec_int_cntl_reg, mec_int_cntl;
3146+ int i, j, k, num_xcc;
2763 3147 
2764 num_xcc = NUM_XCC(adev->gfx.xcc_mask);3148 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2765 switch (state) {3149 switch (state) {
2766 case AMDGPU_IRQ_STATE_DISABLE:3150 case AMDGPU_IRQ_STATE_DISABLE:
2767 case AMDGPU_IRQ_STATE_ENABLE:3151 case AMDGPU_IRQ_STATE_ENABLE:
2768- for (i = 0; i < num_xcc; i++)3152+ for (i = 0; i < num_xcc; i++) {
2769 WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,3153 WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
2770- PRIV_REG_INT_ENABLE,3154+ PRIV_REG_INT_ENABLE,
2771- state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);3155+ state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
3156+ for (j = 0; j < adev->gfx.mec.num_mec; j++) {
3157+ for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
3158+ /* MECs start at 1 */
3159+ mec_int_cntl_reg = gfx_v9_4_3_get_cpc_int_cntl(adev, i, j + 1, k);
3160+ 
3161+ if (mec_int_cntl_reg) {
3162+ mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, i);
3163+ mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3164+ PRIV_REG_INT_ENABLE,
3165+ state == AMDGPU_IRQ_STATE_ENABLE ?
3166+ 1 : 0);
3167+ WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, i);
3168+ }
3169+ }
3170+ }
3171+ }
3172+ break;
3173+ default:
3174+ break;
3175+ }
3176+ 
3177+ return 0;
3178+}
3179+ 
3180+static int gfx_v9_4_3_set_bad_op_fault_state(struct amdgpu_device *adev,
3181+ struct amdgpu_irq_src *source,
3182+ unsigned type,
3183+ enum amdgpu_interrupt_state state)
3184+{
3185+ u32 mec_int_cntl_reg, mec_int_cntl;
3186+ int i, j, k, num_xcc;
3187+ 
3188+ num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3189+ switch (state) {
3190+ case AMDGPU_IRQ_STATE_DISABLE:
3191+ case AMDGPU_IRQ_STATE_ENABLE:
3192+ for (i = 0; i < num_xcc; i++) {
3193+ WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
3194+ OPCODE_ERROR_INT_ENABLE,
3195+ state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
3196+ for (j = 0; j < adev->gfx.mec.num_mec; j++) {
3197+ for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
3198+ /* MECs start at 1 */
3199+ mec_int_cntl_reg = gfx_v9_4_3_get_cpc_int_cntl(adev, i, j + 1, k);
3200+ 
3201+ if (mec_int_cntl_reg) {
3202+ mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, i);
3203+ mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3204+ OPCODE_ERROR_INT_ENABLE,
3205+ state == AMDGPU_IRQ_STATE_ENABLE ?
3206+ 1 : 0);
3207+ WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, i);
3208+ }
3209+ }
3210+ }
3211+ }
2772 break;3212 break;
2773 default:3213 default:
2774 break;3214 break;
@@ -2929,6 +3369,15 @@ static int gfx_v9_4_3_priv_reg_irq(struct amdgpu_device *adev,
2929 return 0;3369 return 0;
2930}3370}
2931 3371 
3372+static int gfx_v9_4_3_bad_op_irq(struct amdgpu_device *adev,
3373+ struct amdgpu_irq_src *source,
3374+ struct amdgpu_iv_entry *entry)
3375+{
3376+ DRM_ERROR("Illegal opcode in command stream\n");
3377+ gfx_v9_4_3_fault(adev, entry);
3378+ return 0;
3379+}
3380+ 
2932static int gfx_v9_4_3_priv_inst_irq(struct amdgpu_device *adev,3381static int gfx_v9_4_3_priv_inst_irq(struct amdgpu_device *adev,
2933 struct amdgpu_irq_src *source,3382 struct amdgpu_irq_src *source,
2934 struct amdgpu_iv_entry *entry)3383 struct amdgpu_iv_entry *entry)
@@ -3015,6 +3464,178 @@ static void gfx_v9_4_3_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
3015 }3464 }
3016}3465}
3017 3466 
3467+static int gfx_v9_4_3_unmap_done(struct amdgpu_device *adev, uint32_t me,
3468+ uint32_t pipe, uint32_t queue,
3469+ uint32_t xcc_id)
3470+{
3471+ int i, r;
3472+ /* make sure dequeue is complete*/
3473+ gfx_v9_4_3_xcc_set_safe_mode(adev, xcc_id);
3474+ mutex_lock(&adev->srbm_mutex);
3475+ soc15_grbm_select(adev, me, pipe, queue, 0, GET_INST(GC, xcc_id));
3476+ for (i = 0; i < adev->usec_timeout; i++) {
3477+ if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
3478+ break;
3479+ udelay(1);
3480+ }
3481+ if (i >= adev->usec_timeout)
3482+ r = -ETIMEDOUT;
3483+ else
3484+ r = 0;
3485+ soc15_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
3486+ mutex_unlock(&adev->srbm_mutex);
3487+ gfx_v9_4_3_xcc_unset_safe_mode(adev, xcc_id);
3488+ 
3489+ return r;
3490+ 
3491+}
3492+ 
3493+static bool gfx_v9_4_3_pipe_reset_support(struct amdgpu_device *adev)
3494+{
3495+ if (!!(adev->gfx.compute_supported_reset & AMDGPU_RESET_TYPE_PER_PIPE))
3496+ return true;
3497+ else
3498+ dev_warn_once(adev->dev, "Please use the latest MEC version to see whether support pipe reset\n");
3499+ 
3500+ return false;
3501+}
3502+ 
3503+static int gfx_v9_4_3_reset_hw_pipe(struct amdgpu_ring *ring)
3504+{
3505+ struct amdgpu_device *adev = ring->adev;
3506+ uint32_t reset_pipe, clean_pipe;
3507+ int r;
3508+ 
3509+ if (!gfx_v9_4_3_pipe_reset_support(adev))
3510+ return -EINVAL;
3511+ 
3512+ gfx_v9_4_3_xcc_set_safe_mode(adev, ring->xcc_id);
3513+ mutex_lock(&adev->srbm_mutex);
3514+ 
3515+ reset_pipe = RREG32_SOC15(GC, GET_INST(GC, ring->xcc_id), regCP_MEC_CNTL);
3516+ clean_pipe = reset_pipe;
3517+ 
3518+ if (ring->me == 1) {
3519+ switch (ring->pipe) {
3520+ case 0:
3521+ reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3522+ MEC_ME1_PIPE0_RESET, 1);
3523+ break;
3524+ case 1:
3525+ reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3526+ MEC_ME1_PIPE1_RESET, 1);
3527+ break;
3528+ case 2:
3529+ reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3530+ MEC_ME1_PIPE2_RESET, 1);
3531+ break;
3532+ case 3:
3533+ reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3534+ MEC_ME1_PIPE3_RESET, 1);
3535+ break;
3536+ default:
3537+ break;
3538+ }
3539+ } else {
3540+ if (ring->pipe)
3541+ reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3542+ MEC_ME2_PIPE1_RESET, 1);
3543+ else
3544+ reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3545+ MEC_ME2_PIPE0_RESET, 1);
3546+ }
3547+ 
3548+ WREG32_SOC15(GC, GET_INST(GC, ring->xcc_id), regCP_MEC_CNTL, reset_pipe);
3549+ WREG32_SOC15(GC, GET_INST(GC, ring->xcc_id), regCP_MEC_CNTL, clean_pipe);
3550+ mutex_unlock(&adev->srbm_mutex);
3551+ gfx_v9_4_3_xcc_unset_safe_mode(adev, ring->xcc_id);
3552+ 
3553+ r = gfx_v9_4_3_unmap_done(adev, ring->me, ring->pipe, ring->queue, ring->xcc_id);
3554+ return r;
3555+}
3556+ 
3557+static int gfx_v9_4_3_reset_kcq(struct amdgpu_ring *ring,
3558+ unsigned int vmid,
3559+ struct amdgpu_fence *timedout_fence)
3560+{
3561+ struct amdgpu_device *adev = ring->adev;
3562+ struct amdgpu_kiq *kiq = &adev->gfx.kiq[ring->xcc_id];
3563+ struct amdgpu_ring *kiq_ring = &kiq->ring;
3564+ int reset_mode = AMDGPU_RESET_TYPE_PER_QUEUE;
3565+ unsigned long flags;
3566+ int r;
3567+ 
3568+ if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
3569+ return -EINVAL;
3570+ 
3571+ amdgpu_ring_reset_helper_begin(ring, timedout_fence);
3572+ 
3573+ spin_lock_irqsave(&kiq->ring_lock, flags);
3574+ 
3575+ if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
3576+ spin_unlock_irqrestore(&kiq->ring_lock, flags);
3577+ return -ENOMEM;
3578+ }
3579+ 
3580+ kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,
3581+ 0, 0);
3582+ amdgpu_ring_commit(kiq_ring);
3583+ 
3584+ spin_unlock_irqrestore(&kiq->ring_lock, flags);
3585+ 
3586+ r = amdgpu_ring_test_ring(kiq_ring);
3587+ if (r) {
3588+ dev_err(adev->dev, "kiq ring test failed after ring: %s queue reset\n",
3589+ ring->name);
3590+ goto pipe_reset;
3591+ }
3592+ 
3593+ r = gfx_v9_4_3_unmap_done(adev, ring->me, ring->pipe, ring->queue, ring->xcc_id);
3594+ if (r)
3595+ dev_err(adev->dev, "fail to wait on hqd deactive and will try pipe reset\n");
3596+ 
3597+pipe_reset:
3598+ if (r) {
3599+ if (!(adev->gfx.compute_supported_reset & AMDGPU_RESET_TYPE_PER_PIPE))
3600+ return -EOPNOTSUPP;
3601+ r = gfx_v9_4_3_reset_hw_pipe(ring);
3602+ reset_mode = AMDGPU_RESET_TYPE_PER_PIPE;
3603+ dev_info(adev->dev, "ring: %s pipe reset :%s\n", ring->name,
3604+ r ? "failed" : "successfully");
3605+ if (r)
3606+ return r;
3607+ }
3608+ 
3609+ gfx_v9_4_3_xcc_kcq_init_queue(ring, ring->xcc_id, true);
3610+ 
3611+ spin_lock_irqsave(&kiq->ring_lock, flags);
3612+ r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
3613+ if (r) {
3614+ spin_unlock_irqrestore(&kiq->ring_lock, flags);
3615+ return -ENOMEM;
3616+ }
3617+ kiq->pmf->kiq_map_queues(kiq_ring, ring);
3618+ amdgpu_ring_commit(kiq_ring);
3619+ r = amdgpu_ring_test_ring(kiq_ring);
3620+ spin_unlock_irqrestore(&kiq->ring_lock, flags);
3621+ if (r) {
3622+ if (reset_mode == AMDGPU_RESET_TYPE_PER_QUEUE)
3623+ goto pipe_reset;
3624+ 
3625+ dev_err(adev->dev, "fail to remap queue\n");
3626+ return r;
3627+ }
3628+ 
3629+ if (reset_mode == AMDGPU_RESET_TYPE_PER_QUEUE) {
3630+ r = amdgpu_ring_test_ring(ring);
3631+ if (r)
3632+ goto pipe_reset;
3633+ }
3634+ 
3635+ 
3636+ return amdgpu_ring_reset_helper_end(ring, timedout_fence);
3637+}
3638+ 
3018enum amdgpu_gfx_cp_ras_mem_id {3639enum amdgpu_gfx_cp_ras_mem_id {
3019 AMDGPU_GFX_CP_MEM1 = 1,3640 AMDGPU_GFX_CP_MEM1 = 1,
3020 AMDGPU_GFX_CP_MEM2,3641 AMDGPU_GFX_CP_MEM2,
@@ -3653,19 +4274,19 @@ static const struct amdgpu_gfx_ras_reg_entry gfx_v9_4_3_ce_reg_list[] = {
3653 AMDGPU_GFX_GC_CANE_MEM, 1},4274 AMDGPU_GFX_GC_CANE_MEM, 1},
3654 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSPI_CE_ERR_STATUS_LO, regSPI_CE_ERR_STATUS_HI),4275 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSPI_CE_ERR_STATUS_LO, regSPI_CE_ERR_STATUS_HI),
3655 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SPI"},4276 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SPI"},
3656- AMDGPU_GFX_SPI_MEM, 8},4277+ AMDGPU_GFX_SPI_MEM, 1},
3657 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP0_CE_ERR_STATUS_LO, regSP0_CE_ERR_STATUS_HI),4278 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP0_CE_ERR_STATUS_LO, regSP0_CE_ERR_STATUS_HI),
3658 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP0"},4279 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP0"},
3659- AMDGPU_GFX_SP_MEM, 1},4280+ AMDGPU_GFX_SP_MEM, 4},
3660 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP1_CE_ERR_STATUS_LO, regSP1_CE_ERR_STATUS_HI),4281 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP1_CE_ERR_STATUS_LO, regSP1_CE_ERR_STATUS_HI),
3661 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP1"},4282 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP1"},
3662- AMDGPU_GFX_SP_MEM, 1},4283+ AMDGPU_GFX_SP_MEM, 4},
3663 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQ_CE_ERR_STATUS_LO, regSQ_CE_ERR_STATUS_HI),4284 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQ_CE_ERR_STATUS_LO, regSQ_CE_ERR_STATUS_HI),
3664 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQ"},4285 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQ"},
3665- AMDGPU_GFX_SQ_MEM, 8},4286+ AMDGPU_GFX_SQ_MEM, 4},
3666 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQC_CE_EDC_LO, regSQC_CE_EDC_HI),4287 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQC_CE_EDC_LO, regSQC_CE_EDC_HI),
3667 5, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQC"},4288 5, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQC"},
3668- AMDGPU_GFX_SQC_MEM, 8},4289+ AMDGPU_GFX_SQC_MEM, 4},
3669 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCX_CE_ERR_STATUS_LO, regTCX_CE_ERR_STATUS_HI),4290 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCX_CE_ERR_STATUS_LO, regTCX_CE_ERR_STATUS_HI),
3670 2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCX"},4291 2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCX"},
3671 AMDGPU_GFX_TCX_MEM, 1},4292 AMDGPU_GFX_TCX_MEM, 1},
@@ -3674,22 +4295,22 @@ static const struct amdgpu_gfx_ras_reg_entry gfx_v9_4_3_ce_reg_list[] = {
3674 AMDGPU_GFX_TCC_MEM, 1},4295 AMDGPU_GFX_TCC_MEM, 1},
3675 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTA_CE_EDC_LO, regTA_CE_EDC_HI),4296 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTA_CE_EDC_LO, regTA_CE_EDC_HI),
3676 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TA"},4297 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TA"},
3677- AMDGPU_GFX_TA_MEM, 8},4298+ AMDGPU_GFX_TA_MEM, 4},
3678 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCI_CE_EDC_LO_REG, regTCI_CE_EDC_HI_REG),4299 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCI_CE_EDC_LO_REG, regTCI_CE_EDC_HI_REG),
3679- 31, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCI"},4300+ 27, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCI"},
3680 AMDGPU_GFX_TCI_MEM, 1},4301 AMDGPU_GFX_TCI_MEM, 1},
3681 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCP_CE_EDC_LO_REG, regTCP_CE_EDC_HI_REG),4302 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCP_CE_EDC_LO_REG, regTCP_CE_EDC_HI_REG),
3682 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCP"},4303 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCP"},
3683- AMDGPU_GFX_TCP_MEM, 8},4304+ AMDGPU_GFX_TCP_MEM, 4},
3684 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTD_CE_EDC_LO, regTD_CE_EDC_HI),4305 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTD_CE_EDC_LO, regTD_CE_EDC_HI),
3685 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TD"},4306 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TD"},
3686- AMDGPU_GFX_TD_MEM, 8},4307+ AMDGPU_GFX_TD_MEM, 4},
3687 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regGCEA_CE_ERR_STATUS_LO, regGCEA_CE_ERR_STATUS_HI),4308 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regGCEA_CE_ERR_STATUS_LO, regGCEA_CE_ERR_STATUS_HI),
3688 16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GCEA"},4309 16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GCEA"},
3689 AMDGPU_GFX_GCEA_MEM, 1},4310 AMDGPU_GFX_GCEA_MEM, 1},
3690 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regLDS_CE_ERR_STATUS_LO, regLDS_CE_ERR_STATUS_HI),4311 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regLDS_CE_ERR_STATUS_LO, regLDS_CE_ERR_STATUS_HI),
3691 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "LDS"},4312 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "LDS"},
3692- AMDGPU_GFX_LDS_MEM, 1},4313+ AMDGPU_GFX_LDS_MEM, 4},
3693};4314};
3694 4315 
3695static const struct amdgpu_gfx_ras_reg_entry gfx_v9_4_3_ue_reg_list[] = {4316static const struct amdgpu_gfx_ras_reg_entry gfx_v9_4_3_ue_reg_list[] = {
@@ -3713,19 +4334,19 @@ static const struct amdgpu_gfx_ras_reg_entry gfx_v9_4_3_ue_reg_list[] = {
3713 AMDGPU_GFX_GC_CANE_MEM, 1},4334 AMDGPU_GFX_GC_CANE_MEM, 1},
3714 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSPI_UE_ERR_STATUS_LO, regSPI_UE_ERR_STATUS_HI),4335 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSPI_UE_ERR_STATUS_LO, regSPI_UE_ERR_STATUS_HI),
3715 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SPI"},4336 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SPI"},
3716- AMDGPU_GFX_SPI_MEM, 8},4337+ AMDGPU_GFX_SPI_MEM, 1},
3717 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP0_UE_ERR_STATUS_LO, regSP0_UE_ERR_STATUS_HI),4338 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP0_UE_ERR_STATUS_LO, regSP0_UE_ERR_STATUS_HI),
3718 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP0"},4339 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP0"},
3719- AMDGPU_GFX_SP_MEM, 1},4340+ AMDGPU_GFX_SP_MEM, 4},
3720 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP1_UE_ERR_STATUS_LO, regSP1_UE_ERR_STATUS_HI),4341 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP1_UE_ERR_STATUS_LO, regSP1_UE_ERR_STATUS_HI),
3721 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP1"},4342 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP1"},
3722- AMDGPU_GFX_SP_MEM, 1},4343+ AMDGPU_GFX_SP_MEM, 4},
3723 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQ_UE_ERR_STATUS_LO, regSQ_UE_ERR_STATUS_HI),4344 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQ_UE_ERR_STATUS_LO, regSQ_UE_ERR_STATUS_HI),
3724 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQ"},4345 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQ"},
3725- AMDGPU_GFX_SQ_MEM, 8},4346+ AMDGPU_GFX_SQ_MEM, 4},
3726 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQC_UE_EDC_LO, regSQC_UE_EDC_HI),4347 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQC_UE_EDC_LO, regSQC_UE_EDC_HI),
3727 5, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQC"},4348 5, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQC"},
3728- AMDGPU_GFX_SQC_MEM, 8},4349+ AMDGPU_GFX_SQC_MEM, 4},
3729 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCX_UE_ERR_STATUS_LO, regTCX_UE_ERR_STATUS_HI),4350 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCX_UE_ERR_STATUS_LO, regTCX_UE_ERR_STATUS_HI),
3730 2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCX"},4351 2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCX"},
3731 AMDGPU_GFX_TCX_MEM, 1},4352 AMDGPU_GFX_TCX_MEM, 1},
@@ -3734,16 +4355,16 @@ static const struct amdgpu_gfx_ras_reg_entry gfx_v9_4_3_ue_reg_list[] = {
3734 AMDGPU_GFX_TCC_MEM, 1},4355 AMDGPU_GFX_TCC_MEM, 1},
3735 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTA_UE_EDC_LO, regTA_UE_EDC_HI),4356 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTA_UE_EDC_LO, regTA_UE_EDC_HI),
3736 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TA"},4357 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TA"},
3737- AMDGPU_GFX_TA_MEM, 8},4358+ AMDGPU_GFX_TA_MEM, 4},
3738 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCI_UE_EDC_LO_REG, regTCI_UE_EDC_HI_REG),4359 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCI_UE_EDC_LO_REG, regTCI_UE_EDC_HI_REG),
3739- 31, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCI"},4360+ 27, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCI"},
3740 AMDGPU_GFX_TCI_MEM, 1},4361 AMDGPU_GFX_TCI_MEM, 1},
3741 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCP_UE_EDC_LO_REG, regTCP_UE_EDC_HI_REG),4362 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCP_UE_EDC_LO_REG, regTCP_UE_EDC_HI_REG),
3742 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCP"},4363 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCP"},
3743- AMDGPU_GFX_TCP_MEM, 8},4364+ AMDGPU_GFX_TCP_MEM, 4},
3744 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTD_UE_EDC_LO, regTD_UE_EDC_HI),4365 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTD_UE_EDC_LO, regTD_UE_EDC_HI),
3745 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TD"},4366 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TD"},
3746- AMDGPU_GFX_TD_MEM, 8},4367+ AMDGPU_GFX_TD_MEM, 4},
3747 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCA_UE_ERR_STATUS_LO, regTCA_UE_ERR_STATUS_HI),4368 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCA_UE_ERR_STATUS_LO, regTCA_UE_ERR_STATUS_HI),
3748 2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCA"},4369 2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCA"},
3749 AMDGPU_GFX_TCA_MEM, 1},4370 AMDGPU_GFX_TCA_MEM, 1},
@@ -3752,11 +4373,7 @@ static const struct amdgpu_gfx_ras_reg_entry gfx_v9_4_3_ue_reg_list[] = {
3752 AMDGPU_GFX_GCEA_MEM, 1},4373 AMDGPU_GFX_GCEA_MEM, 1},
3753 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regLDS_UE_ERR_STATUS_LO, regLDS_UE_ERR_STATUS_HI),4374 {{AMDGPU_RAS_REG_ENTRY(GC, 0, regLDS_UE_ERR_STATUS_LO, regLDS_UE_ERR_STATUS_HI),
3754 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "LDS"},4375 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "LDS"},
3755- AMDGPU_GFX_LDS_MEM, 1},4376+ AMDGPU_GFX_LDS_MEM, 4},
3756-};
3757- 
3758-static const struct soc15_reg_entry gfx_v9_4_3_ea_err_status_regs = {
3759- SOC15_REG_ENTRY(GC, 0, regGCEA_ERR_STATUS), 0, 1, 16
3760};4377};
3761 4378 
3762static void gfx_v9_4_3_inst_query_ras_err_count(struct amdgpu_device *adev,4379static void gfx_v9_4_3_inst_query_ras_err_count(struct amdgpu_device *adev,
@@ -3766,6 +4383,12 @@ static void gfx_v9_4_3_inst_query_ras_err_count(struct amdgpu_device *adev,
3766 unsigned long ce_count = 0, ue_count = 0;4383 unsigned long ce_count = 0, ue_count = 0;
3767 uint32_t i, j, k;4384 uint32_t i, j, k;
3768 4385 
4386+ /* NOTE: convert xcc_id to physical XCD ID (XCD0 or XCD1) */
4387+ struct amdgpu_smuio_mcm_config_info mcm_info = {
4388+ .socket_id = adev->smuio.funcs->get_socket_id(adev),
4389+ .die_id = xcc_id & 0x01 ? 1 : 0,
4390+ };
4391+ 
3769 mutex_lock(&adev->grbm_idx_mutex);4392 mutex_lock(&adev->grbm_idx_mutex);
3770 4393 
3771 for (i = 0; i < ARRAY_SIZE(gfx_v9_4_3_ce_reg_list); i++) {4394 for (i = 0; i < ARRAY_SIZE(gfx_v9_4_3_ce_reg_list); i++) {
@@ -3797,6 +4420,27 @@ static void gfx_v9_4_3_inst_query_ras_err_count(struct amdgpu_device *adev,
3797 }4420 }
3798 }4421 }
3799 4422 
4423+ /* handle extra register entries of UE */
4424+ for (; i < ARRAY_SIZE(gfx_v9_4_3_ue_reg_list); i++) {
4425+ for (j = 0; j < gfx_v9_4_3_ue_reg_list[i].se_num; j++) {
4426+ for (k = 0; k < gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst; k++) {
4427+ /* no need to select if instance number is 1 */
4428+ if (gfx_v9_4_3_ue_reg_list[i].se_num > 1 ||
4429+ gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst > 1)
4430+ gfx_v9_4_3_xcc_select_se_sh(adev, j, 0, k, xcc_id);
4431+ 
4432+ amdgpu_ras_inst_query_ras_error_count(adev,
4433+ &(gfx_v9_4_3_ue_reg_list[i].reg_entry),
4434+ 1,
4435+ gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].mem_id_ent,
4436+ gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].size,
4437+ GET_INST(GC, xcc_id),
4438+ AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
4439+ &ue_count);
4440+ }
4441+ }
4442+ }
4443+ 
3800 gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff,4444 gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff,
3801 xcc_id);4445 xcc_id);
3802 mutex_unlock(&adev->grbm_idx_mutex);4446 mutex_unlock(&adev->grbm_idx_mutex);
@@ -3804,8 +4448,8 @@ static void gfx_v9_4_3_inst_query_ras_err_count(struct amdgpu_device *adev,
3804 /* the caller should make sure initialize value of4448 /* the caller should make sure initialize value of
3805 * err_data->ue_count and err_data->ce_count4449 * err_data->ue_count and err_data->ce_count
3806 */4450 */
3807- err_data->ce_count += ce_count;4451+ amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count);
3808- err_data->ue_count += ue_count;4452+ amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, ce_count);
3809}4453}
3810 4454 
3811static void gfx_v9_4_3_inst_reset_ras_err_count(struct amdgpu_device *adev,4455static void gfx_v9_4_3_inst_reset_ras_err_count(struct amdgpu_device *adev,
@@ -3836,217 +4480,37 @@ static void gfx_v9_4_3_inst_reset_ras_err_count(struct amdgpu_device *adev,
3836 }4480 }
3837 }4481 }
3838 4482 
3839- gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff,4483+ /* handle extra register entries of UE */
3840- xcc_id);4484+ for (; i < ARRAY_SIZE(gfx_v9_4_3_ue_reg_list); i++) {
3841- mutex_unlock(&adev->grbm_idx_mutex);4485+ for (j = 0; j < gfx_v9_4_3_ue_reg_list[i].se_num; j++) {
3842-}4486+ for (k = 0; k < gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst; k++) {
4487+ /* no need to select if instance number is 1 */
4488+ if (gfx_v9_4_3_ue_reg_list[i].se_num > 1 ||
4489+ gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst > 1)
4490+ gfx_v9_4_3_xcc_select_se_sh(adev, j, 0, k, xcc_id);
3843 4491 
3844-static void gfx_v9_4_3_inst_query_ea_err_status(struct amdgpu_device *adev,4492+ amdgpu_ras_inst_reset_ras_error_count(adev,
3845- int xcc_id)4493+ &(gfx_v9_4_3_ue_reg_list[i].reg_entry),
3846-{4494+ 1,
3847- uint32_t i, j;4495+ GET_INST(GC, xcc_id));
3848- uint32_t reg_value;
3849- 
3850- mutex_lock(&adev->grbm_idx_mutex);
3851- 
3852- for (i = 0; i < gfx_v9_4_3_ea_err_status_regs.se_num; i++) {
3853- for (j = 0; j < gfx_v9_4_3_ea_err_status_regs.instance; j++) {
3854- gfx_v9_4_3_xcc_select_se_sh(adev, i, 0, j, xcc_id);
3855- reg_value = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
3856- regGCEA_ERR_STATUS);
3857- if (REG_GET_FIELD(reg_value, GCEA_ERR_STATUS, SDP_RDRSP_STATUS) ||
3858- REG_GET_FIELD(reg_value, GCEA_ERR_STATUS, SDP_WRRSP_STATUS) ||
3859- REG_GET_FIELD(reg_value, GCEA_ERR_STATUS, SDP_RDRSP_DATAPARITY_ERROR)) {
3860- dev_warn(adev->dev,
3861- "GCEA err detected at instance: %d, status: 0x%x!\n",
3862- j, reg_value);
3863- }
3864- /* clear after read */
3865- reg_value = REG_SET_FIELD(reg_value, GCEA_ERR_STATUS,
3866- CLEAR_ERROR_STATUS, 0x1);
3867- WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGCEA_ERR_STATUS,
3868- reg_value);
3869- }
3870- }
3871- 
3872- gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff,
3873- xcc_id);
3874- mutex_unlock(&adev->grbm_idx_mutex);
3875-}
3876- 
3877-static void gfx_v9_4_3_inst_query_utc_err_status(struct amdgpu_device *adev,
3878- int xcc_id)
3879-{
3880- uint32_t data;
3881- 
3882- data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regUTCL2_MEM_ECC_STATUS);
3883- if (data) {
3884- dev_warn(adev->dev, "GFX UTCL2 Mem Ecc Status: 0x%x!\n", data);
3885- WREG32_SOC15(GC, GET_INST(GC, xcc_id), regUTCL2_MEM_ECC_STATUS, 0x3);
3886- }
3887- 
3888- data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regVML2_MEM_ECC_STATUS);
3889- if (data) {
3890- dev_warn(adev->dev, "GFX VML2 Mem Ecc Status: 0x%x!\n", data);
3891- WREG32_SOC15(GC, GET_INST(GC, xcc_id), regVML2_MEM_ECC_STATUS, 0x3);
3892- }
3893- 
3894- data = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
3895- regVML2_WALKER_MEM_ECC_STATUS);
3896- if (data) {
3897- dev_warn(adev->dev, "GFX VML2 Walker Mem Ecc Status: 0x%x!\n", data);
3898- WREG32_SOC15(GC, GET_INST(GC, xcc_id), regVML2_WALKER_MEM_ECC_STATUS,
3899- 0x3);
3900- }
3901-}
3902- 
3903-static void gfx_v9_4_3_log_cu_timeout_status(struct amdgpu_device *adev,
3904- uint32_t status, int xcc_id)
3905-{
3906- struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
3907- uint32_t i, simd, wave;
3908- uint32_t wave_status;
3909- uint32_t wave_pc_lo, wave_pc_hi;
3910- uint32_t wave_exec_lo, wave_exec_hi;
3911- uint32_t wave_inst_dw0, wave_inst_dw1;
3912- uint32_t wave_ib_sts;
3913- 
3914- for (i = 0; i < 32; i++) {
3915- if (!((i << 1) & status))
3916- continue;
3917- 
3918- simd = i / cu_info->max_waves_per_simd;
3919- wave = i % cu_info->max_waves_per_simd;
3920- 
3921- wave_status = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_STATUS);
3922- wave_pc_lo = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_PC_LO);
3923- wave_pc_hi = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_PC_HI);
3924- wave_exec_lo =
3925- wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_EXEC_LO);
3926- wave_exec_hi =
3927- wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_EXEC_HI);
3928- wave_inst_dw0 =
3929- wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_INST_DW0);
3930- wave_inst_dw1 =
3931- wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_INST_DW1);
3932- wave_ib_sts = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_IB_STS);
3933- 
3934- dev_info(
3935- adev->dev,
3936- "\t SIMD %d, Wave %d: status 0x%x, pc 0x%llx, exec 0x%llx, inst 0x%llx, ib_sts 0x%x\n",
3937- simd, wave, wave_status,
3938- ((uint64_t)wave_pc_hi << 32 | wave_pc_lo),
3939- ((uint64_t)wave_exec_hi << 32 | wave_exec_lo),
3940- ((uint64_t)wave_inst_dw1 << 32 | wave_inst_dw0),
3941- wave_ib_sts);
3942- }
3943-}
3944- 
3945-static void gfx_v9_4_3_inst_query_sq_timeout_status(struct amdgpu_device *adev,
3946- int xcc_id)
3947-{
3948- uint32_t se_idx, sh_idx, cu_idx;
3949- uint32_t status;
3950- 
3951- mutex_lock(&adev->grbm_idx_mutex);
3952- for (se_idx = 0; se_idx < adev->gfx.config.max_shader_engines; se_idx++) {
3953- for (sh_idx = 0; sh_idx < adev->gfx.config.max_sh_per_se; sh_idx++) {
3954- for (cu_idx = 0; cu_idx < adev->gfx.config.max_cu_per_sh; cu_idx++) {
3955- gfx_v9_4_3_xcc_select_se_sh(adev, se_idx, sh_idx,
3956- cu_idx, xcc_id);
3957- status = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
3958- regSQ_TIMEOUT_STATUS);
3959- if (status != 0) {
3960- dev_info(
3961- adev->dev,
3962- "GFX Watchdog Timeout: SE %d, SH %d, CU %d\n",
3963- se_idx, sh_idx, cu_idx);
3964- gfx_v9_4_3_log_cu_timeout_status(
3965- adev, status, xcc_id);
3966- }
3967- /* clear old status */
3968- WREG32_SOC15(GC, GET_INST(GC, xcc_id),
3969- regSQ_TIMEOUT_STATUS, 0);
3970 }4496 }
3971 }4497 }
3972 }4498 }
4499+ 
3973 gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff,4500 gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff,
3974 xcc_id);4501 xcc_id);
3975 mutex_unlock(&adev->grbm_idx_mutex);4502 mutex_unlock(&adev->grbm_idx_mutex);
3976}4503}
3977 4504 
3978-static void gfx_v9_4_3_inst_query_ras_err_status(struct amdgpu_device *adev,
3979- void *ras_error_status, int xcc_id)
3980-{
3981- gfx_v9_4_3_inst_query_ea_err_status(adev, xcc_id);
3982- gfx_v9_4_3_inst_query_utc_err_status(adev, xcc_id);
3983- gfx_v9_4_3_inst_query_sq_timeout_status(adev, xcc_id);
3984-}
3985- 
3986-static void gfx_v9_4_3_inst_reset_utc_err_status(struct amdgpu_device *adev,
3987- int xcc_id)
3988-{
3989- WREG32_SOC15(GC, GET_INST(GC, xcc_id), regUTCL2_MEM_ECC_STATUS, 0x3);
3990- WREG32_SOC15(GC, GET_INST(GC, xcc_id), regVML2_MEM_ECC_STATUS, 0x3);
3991- WREG32_SOC15(GC, GET_INST(GC, xcc_id), regVML2_WALKER_MEM_ECC_STATUS, 0x3);
3992-}
3993- 
3994-static void gfx_v9_4_3_inst_reset_ea_err_status(struct amdgpu_device *adev,
3995- int xcc_id)
3996-{
3997- uint32_t i, j;
3998- uint32_t value;
3999- 
4000- mutex_lock(&adev->grbm_idx_mutex);
4001- for (i = 0; i < gfx_v9_4_3_ea_err_status_regs.se_num; i++) {
4002- for (j = 0; j < gfx_v9_4_3_ea_err_status_regs.instance; j++) {
4003- gfx_v9_4_3_xcc_select_se_sh(adev, i, 0, j, xcc_id);
4004- value = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regGCEA_ERR_STATUS);
4005- value = REG_SET_FIELD(value, GCEA_ERR_STATUS,
4006- CLEAR_ERROR_STATUS, 0x1);
4007- WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGCEA_ERR_STATUS, value);
4008- }
4009- }
4010- gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff,
4011- xcc_id);
4012- mutex_unlock(&adev->grbm_idx_mutex);
4013-}
4014- 
4015-static void gfx_v9_4_3_inst_reset_sq_timeout_status(struct amdgpu_device *adev,
4016- int xcc_id)
4017-{
4018- uint32_t se_idx, sh_idx, cu_idx;
4019- 
4020- mutex_lock(&adev->grbm_idx_mutex);
4021- for (se_idx = 0; se_idx < adev->gfx.config.max_shader_engines; se_idx++) {
4022- for (sh_idx = 0; sh_idx < adev->gfx.config.max_sh_per_se; sh_idx++) {
4023- for (cu_idx = 0; cu_idx < adev->gfx.config.max_cu_per_sh; cu_idx++) {
4024- gfx_v9_4_3_xcc_select_se_sh(adev, se_idx, sh_idx,
4025- cu_idx, xcc_id);
4026- WREG32_SOC15(GC, GET_INST(GC, xcc_id),
4027- regSQ_TIMEOUT_STATUS, 0);
4028- }
4029- }
4030- }
4031- gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff,
4032- xcc_id);
4033- mutex_unlock(&adev->grbm_idx_mutex);
4034-}
4035- 
4036-static void gfx_v9_4_3_inst_reset_ras_err_status(struct amdgpu_device *adev,
4037- void *ras_error_status, int xcc_id)
4038-{
4039- gfx_v9_4_3_inst_reset_utc_err_status(adev, xcc_id);
4040- gfx_v9_4_3_inst_reset_ea_err_status(adev, xcc_id);
4041- gfx_v9_4_3_inst_reset_sq_timeout_status(adev, xcc_id);
4042-}
4043- 
4044static void gfx_v9_4_3_inst_enable_watchdog_timer(struct amdgpu_device *adev,4505static void gfx_v9_4_3_inst_enable_watchdog_timer(struct amdgpu_device *adev,
4045 void *ras_error_status, int xcc_id)4506 void *ras_error_status, int xcc_id)
4046{4507{
4047 uint32_t i;4508 uint32_t i;
4048 uint32_t data;4509 uint32_t data;
4049 4510 
4511+ if (amdgpu_sriov_vf(adev))
4512+ return;
4513+ 
4050 data = RREG32_SOC15(GC, GET_INST(GC, 0), regSQ_TIMEOUT_CONFIG);4514 data = RREG32_SOC15(GC, GET_INST(GC, 0), regSQ_TIMEOUT_CONFIG);
4051 data = REG_SET_FIELD(data, SQ_TIMEOUT_CONFIG, TIMEOUT_FATAL_DISABLE,4515 data = REG_SET_FIELD(data, SQ_TIMEOUT_CONFIG, TIMEOUT_FATAL_DISABLE,
4052 amdgpu_watchdog_timer.timeout_fatal_disable ? 1 : 0);4516 amdgpu_watchdog_timer.timeout_fatal_disable ? 1 : 0);
@@ -4082,21 +4546,166 @@ static void gfx_v9_4_3_reset_ras_error_count(struct amdgpu_device *adev)
4082 amdgpu_gfx_ras_error_func(adev, NULL, gfx_v9_4_3_inst_reset_ras_err_count);4546 amdgpu_gfx_ras_error_func(adev, NULL, gfx_v9_4_3_inst_reset_ras_err_count);
4083}4547}
4084 4548 
4085-static void gfx_v9_4_3_query_ras_error_status(struct amdgpu_device *adev)
4086-{
4087- amdgpu_gfx_ras_error_func(adev, NULL, gfx_v9_4_3_inst_query_ras_err_status);
4088-}
4089- 
4090-static void gfx_v9_4_3_reset_ras_error_status(struct amdgpu_device *adev)
4091-{
4092- amdgpu_gfx_ras_error_func(adev, NULL, gfx_v9_4_3_inst_reset_ras_err_status);
4093-}
4094- 
4095static void gfx_v9_4_3_enable_watchdog_timer(struct amdgpu_device *adev)4549static void gfx_v9_4_3_enable_watchdog_timer(struct amdgpu_device *adev)
4096{4550{
4097 amdgpu_gfx_ras_error_func(adev, NULL, gfx_v9_4_3_inst_enable_watchdog_timer);4551 amdgpu_gfx_ras_error_func(adev, NULL, gfx_v9_4_3_inst_enable_watchdog_timer);
4098}4552}
4099 4553 
4554+static void gfx_v9_4_3_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
4555+{
4556+ /* Header itself is a NOP packet */
4557+ if (num_nop == 1) {
4558+ amdgpu_ring_write(ring, ring->funcs->nop);
4559+ return;
4560+ }
4561+ 
4562+ /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
4563+ amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
4564+ 
4565+ /* Header is at index 0, followed by num_nops - 1 NOP packet's */
4566+ amdgpu_ring_insert_nop(ring, num_nop - 1);
4567+}
4568+ 
4569+static void gfx_v9_4_3_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
4570+{
4571+ struct amdgpu_device *adev = ip_block->adev;
4572+ uint32_t i, j, k;
4573+ uint32_t xcc_id, xcc_offset, inst_offset;
4574+ uint32_t num_xcc, reg, num_inst;
4575+ uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9_4_3);
4576+ 
4577+ if (!adev->gfx.ip_dump_core)
4578+ return;
4579+ 
4580+ num_xcc = NUM_XCC(adev->gfx.xcc_mask);
4581+ drm_printf(p, "Number of Instances:%d\n", num_xcc);
4582+ for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
4583+ xcc_offset = xcc_id * reg_count;
4584+ drm_printf(p, "\nInstance id:%d\n", xcc_id);
4585+ for (i = 0; i < reg_count; i++)
4586+ drm_printf(p, "%-50s \t 0x%08x\n",
4587+ gc_reg_list_9_4_3[i].reg_name,
4588+ adev->gfx.ip_dump_core[xcc_offset + i]);
4589+ }
4590+ 
4591+ /* print compute queue registers for all instances */
4592+ if (!adev->gfx.ip_dump_compute_queues)
4593+ return;
4594+ 
4595+ num_inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
4596+ adev->gfx.mec.num_queue_per_pipe;
4597+ 
4598+ reg_count = ARRAY_SIZE(gc_cp_reg_list_9_4_3);
4599+ drm_printf(p, "\nnum_xcc: %d num_mec: %d num_pipe: %d num_queue: %d\n",
4600+ num_xcc,
4601+ adev->gfx.mec.num_mec,
4602+ adev->gfx.mec.num_pipe_per_mec,
4603+ adev->gfx.mec.num_queue_per_pipe);
4604+ 
4605+ for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
4606+ xcc_offset = xcc_id * reg_count * num_inst;
4607+ inst_offset = 0;
4608+ for (i = 0; i < adev->gfx.mec.num_mec; i++) {
4609+ for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
4610+ for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
4611+ drm_printf(p,
4612+ "\nxcc:%d mec:%d, pipe:%d, queue:%d\n",
4613+ xcc_id, i, j, k);
4614+ for (reg = 0; reg < reg_count; reg++) {
4615+ if (i && gc_cp_reg_list_9_4_3[reg].reg_offset ==
4616+ regCP_MEC_ME1_HEADER_DUMP)
4617+ drm_printf(p,
4618+ "%-50s \t 0x%08x\n",
4619+ "regCP_MEC_ME2_HEADER_DUMP",
4620+ adev->gfx.ip_dump_compute_queues
4621+ [xcc_offset + inst_offset +
4622+ reg]);
4623+ else
4624+ drm_printf(p,
4625+ "%-50s \t 0x%08x\n",
4626+ gc_cp_reg_list_9_4_3[reg].reg_name,
4627+ adev->gfx.ip_dump_compute_queues
4628+ [xcc_offset + inst_offset +
4629+ reg]);
4630+ }
4631+ inst_offset += reg_count;
4632+ }
4633+ }
4634+ }
4635+ }
4636+}
4637+ 
4638+static void gfx_v9_4_3_ip_dump(struct amdgpu_ip_block *ip_block)
4639+{
4640+ struct amdgpu_device *adev = ip_block->adev;
4641+ uint32_t i, j, k;
4642+ uint32_t num_xcc, reg, num_inst;
4643+ uint32_t xcc_id, xcc_offset, inst_offset;
4644+ uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9_4_3);
4645+ 
4646+ if (!adev->gfx.ip_dump_core)
4647+ return;
4648+ 
4649+ num_xcc = NUM_XCC(adev->gfx.xcc_mask);
4650+ 
4651+ for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
4652+ xcc_offset = xcc_id * reg_count;
4653+ for (i = 0; i < reg_count; i++)
4654+ adev->gfx.ip_dump_core[xcc_offset + i] =
4655+ RREG32(SOC15_REG_ENTRY_OFFSET_INST(gc_reg_list_9_4_3[i],
4656+ GET_INST(GC, xcc_id)));
4657+ }
4658+ 
4659+ /* dump compute queue registers for all instances */
4660+ if (!adev->gfx.ip_dump_compute_queues)
4661+ return;
4662+ 
4663+ num_inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
4664+ adev->gfx.mec.num_queue_per_pipe;
4665+ reg_count = ARRAY_SIZE(gc_cp_reg_list_9_4_3);
4666+ mutex_lock(&adev->srbm_mutex);
4667+ for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
4668+ xcc_offset = xcc_id * reg_count * num_inst;
4669+ inst_offset = 0;
4670+ for (i = 0; i < adev->gfx.mec.num_mec; i++) {
4671+ for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
4672+ for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
4673+ /* ME0 is for GFX so start from 1 for CP */
4674+ soc15_grbm_select(adev, 1 + i, j, k, 0,
4675+ GET_INST(GC, xcc_id));
4676+ 
4677+ for (reg = 0; reg < reg_count; reg++) {
4678+ if (i && gc_cp_reg_list_9_4_3[reg].reg_offset ==
4679+ regCP_MEC_ME1_HEADER_DUMP)
4680+ adev->gfx.ip_dump_compute_queues
4681+ [xcc_offset +
4682+ inst_offset + reg] =
4683+ RREG32(SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id),
4684+ regCP_MEC_ME2_HEADER_DUMP));
4685+ else
4686+ adev->gfx.ip_dump_compute_queues
4687+ [xcc_offset +
4688+ inst_offset + reg] =
4689+ RREG32(SOC15_REG_ENTRY_OFFSET_INST(
4690+ gc_cp_reg_list_9_4_3[reg],
4691+ GET_INST(GC, xcc_id)));
4692+ }
4693+ inst_offset += reg_count;
4694+ }
4695+ }
4696+ }
4697+ }
4698+ soc15_grbm_select(adev, 0, 0, 0, 0, 0);
4699+ mutex_unlock(&adev->srbm_mutex);
4700+}
4701+ 
4702+static void gfx_v9_4_3_ring_emit_cleaner_shader(struct amdgpu_ring *ring)
4703+{
4704+ /* Emit the cleaner shader */
4705+ amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));
4706+ amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */
4707+}
4708+ 
4100static const struct amd_ip_funcs gfx_v9_4_3_ip_funcs = {4709static const struct amd_ip_funcs gfx_v9_4_3_ip_funcs = {
4101 .name = "gfx_v9_4_3",4710 .name = "gfx_v9_4_3",
4102 .early_init = gfx_v9_4_3_early_init,4711 .early_init = gfx_v9_4_3_early_init,
@@ -4113,6 +4722,8 @@ static const struct amd_ip_funcs gfx_v9_4_3_ip_funcs = {
4113 .set_clockgating_state = gfx_v9_4_3_set_clockgating_state,4722 .set_clockgating_state = gfx_v9_4_3_set_clockgating_state,
4114 .set_powergating_state = gfx_v9_4_3_set_powergating_state,4723 .set_powergating_state = gfx_v9_4_3_set_powergating_state,
4115 .get_clockgating_state = gfx_v9_4_3_get_clockgating_state,4724 .get_clockgating_state = gfx_v9_4_3_get_clockgating_state,
4725+ .dump_ip_state = gfx_v9_4_3_ip_dump,
4726+ .print_ip_state = gfx_v9_4_3_ip_print,
4116};4727};
4117 4728 
4118static const struct amdgpu_ring_funcs gfx_v9_4_3_ring_funcs_compute = {4729static const struct amdgpu_ring_funcs gfx_v9_4_3_ring_funcs_compute = {
@@ -4134,7 +4745,8 @@ static const struct amdgpu_ring_funcs gfx_v9_4_3_ring_funcs_compute = {
4134 8 + 8 + 8 + /* gfx_v9_4_3_ring_emit_fence x3 for user fence, vm fence */4745 8 + 8 + 8 + /* gfx_v9_4_3_ring_emit_fence x3 for user fence, vm fence */
4135 7 + /* gfx_v9_4_3_emit_mem_sync */4746 7 + /* gfx_v9_4_3_emit_mem_sync */
4136 5 + /* gfx_v9_4_3_emit_wave_limit for updating regSPI_WCL_PIPE_PERCENT_GFX register */4747 5 + /* gfx_v9_4_3_emit_wave_limit for updating regSPI_WCL_PIPE_PERCENT_GFX register */
4137- 15, /* for updating 3 regSPI_WCL_PIPE_PERCENT_CS registers */4748+ 15 + /* for updating 3 regSPI_WCL_PIPE_PERCENT_CS registers */
4749+ 2, /* gfx_v9_4_3_ring_emit_cleaner_shader */
4138 .emit_ib_size = 7, /* gfx_v9_4_3_ring_emit_ib_compute */4750 .emit_ib_size = 7, /* gfx_v9_4_3_ring_emit_ib_compute */
4139 .emit_ib = gfx_v9_4_3_ring_emit_ib_compute,4751 .emit_ib = gfx_v9_4_3_ring_emit_ib_compute,
4140 .emit_fence = gfx_v9_4_3_ring_emit_fence,4752 .emit_fence = gfx_v9_4_3_ring_emit_fence,
@@ -4144,13 +4756,18 @@ static const struct amdgpu_ring_funcs gfx_v9_4_3_ring_funcs_compute = {
4144 .emit_hdp_flush = gfx_v9_4_3_ring_emit_hdp_flush,4756 .emit_hdp_flush = gfx_v9_4_3_ring_emit_hdp_flush,
4145 .test_ring = gfx_v9_4_3_ring_test_ring,4757 .test_ring = gfx_v9_4_3_ring_test_ring,
4146 .test_ib = gfx_v9_4_3_ring_test_ib,4758 .test_ib = gfx_v9_4_3_ring_test_ib,
4147- .insert_nop = amdgpu_ring_insert_nop,4759+ .insert_nop = gfx_v9_4_3_ring_insert_nop,
4148 .pad_ib = amdgpu_ring_generic_pad_ib,4760 .pad_ib = amdgpu_ring_generic_pad_ib,
4149 .emit_wreg = gfx_v9_4_3_ring_emit_wreg,4761 .emit_wreg = gfx_v9_4_3_ring_emit_wreg,
4150 .emit_reg_wait = gfx_v9_4_3_ring_emit_reg_wait,4762 .emit_reg_wait = gfx_v9_4_3_ring_emit_reg_wait,
4151 .emit_reg_write_reg_wait = gfx_v9_4_3_ring_emit_reg_write_reg_wait,4763 .emit_reg_write_reg_wait = gfx_v9_4_3_ring_emit_reg_write_reg_wait,
4764+ .soft_recovery = gfx_v9_4_3_ring_soft_recovery,
4152 .emit_mem_sync = gfx_v9_4_3_emit_mem_sync,4765 .emit_mem_sync = gfx_v9_4_3_emit_mem_sync,
4153 .emit_wave_limit = gfx_v9_4_3_emit_wave_limit,4766 .emit_wave_limit = gfx_v9_4_3_emit_wave_limit,
4767+ .reset = gfx_v9_4_3_reset_kcq,
4768+ .emit_cleaner_shader = gfx_v9_4_3_ring_emit_cleaner_shader,
4769+ .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
4770+ .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
4154};4771};
4155 4772 
4156static const struct amdgpu_ring_funcs gfx_v9_4_3_ring_funcs_kiq = {4773static const struct amdgpu_ring_funcs gfx_v9_4_3_ring_funcs_kiq = {
@@ -4179,6 +4796,7 @@ static const struct amdgpu_ring_funcs gfx_v9_4_3_ring_funcs_kiq = {
4179 .emit_wreg = gfx_v9_4_3_ring_emit_wreg,4796 .emit_wreg = gfx_v9_4_3_ring_emit_wreg,
4180 .emit_reg_wait = gfx_v9_4_3_ring_emit_reg_wait,4797 .emit_reg_wait = gfx_v9_4_3_ring_emit_reg_wait,
4181 .emit_reg_write_reg_wait = gfx_v9_4_3_ring_emit_reg_write_reg_wait,4798 .emit_reg_write_reg_wait = gfx_v9_4_3_ring_emit_reg_write_reg_wait,
4799+ .emit_hdp_flush = gfx_v9_4_3_ring_emit_hdp_flush,
4182};4800};
4183 4801 
4184static void gfx_v9_4_3_set_ring_funcs(struct amdgpu_device *adev)4802static void gfx_v9_4_3_set_ring_funcs(struct amdgpu_device *adev)
@@ -4205,6 +4823,11 @@ static const struct amdgpu_irq_src_funcs gfx_v9_4_3_priv_reg_irq_funcs = {
4205 .process = gfx_v9_4_3_priv_reg_irq,4823 .process = gfx_v9_4_3_priv_reg_irq,
4206};4824};
4207 4825 
4826+static const struct amdgpu_irq_src_funcs gfx_v9_4_3_bad_op_irq_funcs = {
4827+ .set = gfx_v9_4_3_set_bad_op_fault_state,
4828+ .process = gfx_v9_4_3_bad_op_irq,
4829+};
4830+ 
4208static const struct amdgpu_irq_src_funcs gfx_v9_4_3_priv_inst_irq_funcs = {4831static const struct amdgpu_irq_src_funcs gfx_v9_4_3_priv_inst_irq_funcs = {
4209 .set = gfx_v9_4_3_set_priv_inst_fault_state,4832 .set = gfx_v9_4_3_set_priv_inst_fault_state,
4210 .process = gfx_v9_4_3_priv_inst_irq,4833 .process = gfx_v9_4_3_priv_inst_irq,
@@ -4218,6 +4841,9 @@ static void gfx_v9_4_3_set_irq_funcs(struct amdgpu_device *adev)
4218 adev->gfx.priv_reg_irq.num_types = 1;4841 adev->gfx.priv_reg_irq.num_types = 1;
4219 adev->gfx.priv_reg_irq.funcs = &gfx_v9_4_3_priv_reg_irq_funcs;4842 adev->gfx.priv_reg_irq.funcs = &gfx_v9_4_3_priv_reg_irq_funcs;
4220 4843 
4844+ adev->gfx.bad_op_irq.num_types = 1;
4845+ adev->gfx.bad_op_irq.funcs = &gfx_v9_4_3_bad_op_irq_funcs;
4846+ 
4221 adev->gfx.priv_inst_irq.num_types = 1;4847 adev->gfx.priv_inst_irq.num_types = 1;
4222 adev->gfx.priv_inst_irq.funcs = &gfx_v9_4_3_priv_inst_irq_funcs;4848 adev->gfx.priv_inst_irq.funcs = &gfx_v9_4_3_priv_inst_irq_funcs;
4223}4849}
@@ -4230,30 +4856,13 @@ static void gfx_v9_4_3_set_rlc_funcs(struct amdgpu_device *adev)
4230 4856 
4231static void gfx_v9_4_3_set_gds_init(struct amdgpu_device *adev)4857static void gfx_v9_4_3_set_gds_init(struct amdgpu_device *adev)
4232{4858{
4233- /* init asci gds info */4859+ /* 9.4.3 variants removed all the GDS internal memory,
4234- switch (adev->ip_versions[GC_HWIP][0]) {4860+ * only support GWS opcode in kernel, like barrier
4235- case IP_VERSION(9, 4, 3):4861+ * semaphore.etc */
4236- /* 9.4.3 removed all the GDS internal memory,
4237- * only support GWS opcode in kernel, like barrier
4238- * semaphore.etc */
4239- adev->gds.gds_size = 0;
4240- break;
4241- default:
4242- adev->gds.gds_size = 0x10000;
4243- break;
4244- }
4245- 
4246- switch (adev->ip_versions[GC_HWIP][0]) {
4247- case IP_VERSION(9, 4, 3):
4248- /* deprecated for 9.4.3, no usage at all */
4249- adev->gds.gds_compute_max_wave_id = 0;
4250- break;
4251- default:
4252- /* this really depends on the chip */
4253- adev->gds.gds_compute_max_wave_id = 0x7ff;
4254- break;
4255- }
4256 4862 
4863+ /* init asic gds info */
4864+ adev->gds.gds_size = 0;
4865+ adev->gds.gds_compute_max_wave_id = 0;
4257 adev->gds.gws_size = 64;4866 adev->gds.gws_size = 64;
4258 adev->gds.oa_size = 16;4867 adev->gds.oa_size = 16;
4259}4868}
@@ -4365,7 +4974,7 @@ const struct amdgpu_ip_block_version gfx_v9_4_3_ip_block = {
4365 .type = AMD_IP_BLOCK_TYPE_GFX,4974 .type = AMD_IP_BLOCK_TYPE_GFX,
4366 .major = 9,4975 .major = 9,
4367 .minor = 4,4976 .minor = 4,
4368- .rev = 0,4977+ .rev = 3,
4369 .funcs = &gfx_v9_4_3_ip_funcs,4978 .funcs = &gfx_v9_4_3_ip_funcs,
4370};4979};
4371 4980 
@@ -4420,13 +5029,34 @@ struct amdgpu_xcp_ip_funcs gfx_v9_4_3_xcp_funcs = {
4420struct amdgpu_ras_block_hw_ops gfx_v9_4_3_ras_ops = {5029struct amdgpu_ras_block_hw_ops gfx_v9_4_3_ras_ops = {
4421 .query_ras_error_count = &gfx_v9_4_3_query_ras_error_count,5030 .query_ras_error_count = &gfx_v9_4_3_query_ras_error_count,
4422 .reset_ras_error_count = &gfx_v9_4_3_reset_ras_error_count,5031 .reset_ras_error_count = &gfx_v9_4_3_reset_ras_error_count,
4423- .query_ras_error_status = &gfx_v9_4_3_query_ras_error_status,
4424- .reset_ras_error_status = &gfx_v9_4_3_reset_ras_error_status,
4425};5032};
4426 5033 
5034+static int gfx_v9_4_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
5035+{
5036+ int r;
5037+ 
5038+ r = amdgpu_ras_block_late_init(adev, ras_block);
5039+ if (r)
5040+ return r;
5041+ 
5042+ r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__GFX,
5043+ &gfx_v9_4_3_aca_info,
5044+ NULL);
5045+ if (r)
5046+ goto late_fini;
5047+ 
5048+ return 0;
5049+ 
5050+late_fini:
5051+ amdgpu_ras_block_late_fini(adev, ras_block);
5052+ 
5053+ return r;
5054+}
5055+ 
4427struct amdgpu_gfx_ras gfx_v9_4_3_ras = {5056struct amdgpu_gfx_ras gfx_v9_4_3_ras = {
4428 .ras_block = {5057 .ras_block = {
4429 .hw_ops = &gfx_v9_4_3_ras_ops,5058 .hw_ops = &gfx_v9_4_3_ras_ops,
5059+ .ras_late_init = &gfx_v9_4_3_ras_late_init,
4430 },5060 },
4431 .enable_watchdog_timer = &gfx_v9_4_3_enable_watchdog_timer,5061 .enable_watchdog_timer = &gfx_v9_4_3_enable_watchdog_timer,
4432};5062};
Mdrivers/gpu/drm/radeon/radeon_fence.c+49-104
@@ -30,6 +30,7 @@
30 */30 */
31 31 
32#include <linux/atomic.h>32#include <linux/atomic.h>
33+#include <linux/debugfs.h>
33#include <linux/firmware.h>34#include <linux/firmware.h>
34#include <linux/kref.h>35#include <linux/kref.h>
35#include <linux/sched/signal.h>36#include <linux/sched/signal.h>
@@ -45,7 +46,6 @@
45#include "radeon_trace.h"46#include "radeon_trace.h"
46 47 
47/*48/*
48- * Fences
49 * Fences mark an event in the GPUs pipeline and are used49 * Fences mark an event in the GPUs pipeline and are used
50 * for GPU/CPU synchronization. When the fence is written,50 * for GPU/CPU synchronization. When the fence is written,
51 * it is expected that all buffers associated with that fence51 * it is expected that all buffers associated with that fence
@@ -67,10 +67,10 @@
67static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)67static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)
68{68{
69 struct radeon_fence_driver *drv = &rdev->fence_drv[ring];69 struct radeon_fence_driver *drv = &rdev->fence_drv[ring];
70+ 
70 if (likely(rdev->wb.enabled || !drv->scratch_reg)) {71 if (likely(rdev->wb.enabled || !drv->scratch_reg)) {
71- if (drv->cpu_addr) {72+ if (drv->cpu_addr)
72 *drv->cpu_addr = cpu_to_le32(seq);73 *drv->cpu_addr = cpu_to_le32(seq);
73- }
74 } else {74 } else {
75 WREG32(drv->scratch_reg, seq);75 WREG32(drv->scratch_reg, seq);
76 }76 }
@@ -91,11 +91,10 @@ static u32 radeon_fence_read(struct radeon_device *rdev, int ring)
91 u32 seq = 0;91 u32 seq = 0;
92 92 
93 if (likely(rdev->wb.enabled || !drv->scratch_reg)) {93 if (likely(rdev->wb.enabled || !drv->scratch_reg)) {
94- if (drv->cpu_addr) {94+ if (drv->cpu_addr)
95 seq = le32_to_cpu(*drv->cpu_addr);95 seq = le32_to_cpu(*drv->cpu_addr);
96- } else {96+ else
97 seq = lower_32_bits(atomic64_read(&drv->last_seq));97 seq = lower_32_bits(atomic64_read(&drv->last_seq));
98- }
99 } else {98 } else {
100 seq = RREG32(drv->scratch_reg);99 seq = RREG32(drv->scratch_reg);
101 }100 }
@@ -139,9 +138,9 @@ int radeon_fence_emit(struct radeon_device *rdev,
139 138 
140 /* we are protected by the ring emission mutex */139 /* we are protected by the ring emission mutex */
141 *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);140 *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
142- if ((*fence) == NULL) {141+ if ((*fence) == NULL)
143 return -ENOMEM;142 return -ENOMEM;
144- }143+ 
145 (*fence)->rdev = rdev;144 (*fence)->rdev = rdev;
146 (*fence)->seq = seq = ++rdev->fence_drv[ring].sync_seq[ring];145 (*fence)->seq = seq = ++rdev->fence_drv[ring].sync_seq[ring];
147 (*fence)->ring = ring;146 (*fence)->ring = ring;
@@ -163,7 +162,8 @@ int radeon_fence_emit(struct radeon_device *rdev,
163 * for the fence locking itself, so unlocked variants are used for162 * for the fence locking itself, so unlocked variants are used for
164 * fence_signal, and remove_wait_queue.163 * fence_signal, and remove_wait_queue.
165 */164 */
166-static int radeon_fence_check_signaled(wait_queue_entry_t *wait, unsigned mode, int flags, void *key)165+static int radeon_fence_check_signaled(wait_queue_entry_t *wait,
166+ unsigned int mode, int flags, void *key)
167{167{
168 struct radeon_fence *fence;168 struct radeon_fence *fence;
169 u64 seq;169 u64 seq;
@@ -197,7 +197,7 @@ static int radeon_fence_check_signaled(wait_queue_entry_t *wait, unsigned mode,
197static bool radeon_fence_activity(struct radeon_device *rdev, int ring)197static bool radeon_fence_activity(struct radeon_device *rdev, int ring)
198{198{
199 uint64_t seq, last_seq, last_emitted;199 uint64_t seq, last_seq, last_emitted;
200- unsigned count_loop = 0;200+ unsigned int count_loop = 0;
201 bool wake = false;201 bool wake = false;
202 202 
203 /* Note there is a scenario here for an infinite loop but it's203 /* Note there is a scenario here for an infinite loop but it's
@@ -231,9 +231,9 @@ static bool radeon_fence_activity(struct radeon_device *rdev, int ring)
231 seq |= last_emitted & 0xffffffff00000000LL;231 seq |= last_emitted & 0xffffffff00000000LL;
232 }232 }
233 233 
234- if (seq <= last_seq || seq > last_emitted) {234+ if (seq <= last_seq || seq > last_emitted)
235 break;235 break;
236- }236+ 
237 /* If we loop over we don't want to return without237 /* If we loop over we don't want to return without
238 * checking if a fence is signaled as it means that the238 * checking if a fence is signaled as it means that the
239 * seq we just read is different from the previous on.239 * seq we just read is different from the previous on.
@@ -296,8 +296,7 @@ static void radeon_fence_check_lockup(struct work_struct *work)
296 else if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) {296 else if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) {
297 297 
298 /* good news we believe it's a lockup */298 /* good news we believe it's a lockup */
299- dev_warn(rdev->dev, "GPU lockup (current fence id "299+ dev_warn(rdev->dev, "GPU lockup (current fence id 0x%016llx last fence id 0x%016llx on ring %d)\n",
300- "0x%016llx last fence id 0x%016llx on ring %d)\n",
301 (uint64_t)atomic64_read(&fence_drv->last_seq),300 (uint64_t)atomic64_read(&fence_drv->last_seq),
302 fence_drv->sync_seq[ring], ring);301 fence_drv->sync_seq[ring], ring);
303 302 
@@ -338,16 +337,16 @@ void radeon_fence_process(struct radeon_device *rdev, int ring)
338 * radeon_fence_signaled().337 * radeon_fence_signaled().
339 */338 */
340static bool radeon_fence_seq_signaled(struct radeon_device *rdev,339static bool radeon_fence_seq_signaled(struct radeon_device *rdev,
341- u64 seq, unsigned ring)340+ u64 seq, unsigned int ring)
342{341{
343- if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) {342+ if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq)
344 return true;343 return true;
345- }344+ 
346 /* poll new last sequence at least once */345 /* poll new last sequence at least once */
347 radeon_fence_process(rdev, ring);346 radeon_fence_process(rdev, ring);
348- if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) {347+ if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq)
349 return true;348 return true;
350- }349+ 
351 return false;350 return false;
352}351}
353 352 
@@ -355,21 +354,12 @@ static bool radeon_fence_is_signaled(struct dma_fence *f)
355{354{
356 struct radeon_fence *fence = to_radeon_fence(f);355 struct radeon_fence *fence = to_radeon_fence(f);
357 struct radeon_device *rdev = fence->rdev;356 struct radeon_device *rdev = fence->rdev;
358- unsigned ring = fence->ring;357+ unsigned int ring = fence->ring;
359 u64 seq = fence->seq;358 u64 seq = fence->seq;
360 359 
361- if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) {360+ if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq)
362 return true;361 return true;
363- }
364 362 
365- if (down_read_trylock(&rdev->exclusive_lock)) {
366- radeon_fence_process(rdev, ring);
367- up_read(&rdev->exclusive_lock);
368- 
369- if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) {
370- return true;
371- }
372- }
373 return false;363 return false;
374}364}
375 365 
@@ -451,7 +441,7 @@ bool radeon_fence_signaled(struct radeon_fence *fence)
451 */441 */
452static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq)442static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq)
453{443{
454- unsigned i;444+ unsigned int i;
455 445 
456 for (i = 0; i < RADEON_NUM_RINGS; ++i) {446 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
457 if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i))447 if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i))
@@ -549,9 +539,8 @@ long radeon_fence_wait_timeout(struct radeon_fence *fence, bool intr, long timeo
549 539 
550 seq[fence->ring] = fence->seq;540 seq[fence->ring] = fence->seq;
551 r = radeon_fence_wait_seq_timeout(fence->rdev, seq, intr, timeout);541 r = radeon_fence_wait_seq_timeout(fence->rdev, seq, intr, timeout);
552- if (r <= 0) {542+ if (r <= 0)
553 return r;543 return r;
554- }
555 544 
556 dma_fence_signal(&fence->base);545 dma_fence_signal(&fence->base);
557 return r;546 return r;
@@ -571,54 +560,11 @@ long radeon_fence_wait_timeout(struct radeon_fence *fence, bool intr, long timeo
571int radeon_fence_wait(struct radeon_fence *fence, bool intr)560int radeon_fence_wait(struct radeon_fence *fence, bool intr)
572{561{
573 long r = radeon_fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT);562 long r = radeon_fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT);
574- if (r > 0) {563+ 
564+ if (r > 0)
575 return 0;565 return 0;
576- } else {566+ else
577 return r;567 return r;
578- }
579-}
580- 
581-/**
582- * radeon_fence_wait_any - wait for a fence to signal on any ring
583- *
584- * @rdev: radeon device pointer
585- * @fences: radeon fence object(s)
586- * @intr: use interruptable sleep
587- *
588- * Wait for any requested fence to signal (all asics). Fence
589- * array is indexed by ring id. @intr selects whether to use
590- * interruptable (true) or non-interruptable (false) sleep when
591- * waiting for the fences. Used by the suballocator.
592- * Returns 0 if any fence has passed, error for all other cases.
593- */
594-int radeon_fence_wait_any(struct radeon_device *rdev,
595- struct radeon_fence **fences,
596- bool intr)
597-{
598- uint64_t seq[RADEON_NUM_RINGS];
599- unsigned i, num_rings = 0;
600- long r;
601- 
602- for (i = 0; i < RADEON_NUM_RINGS; ++i) {
603- seq[i] = 0;
604- 
605- if (!fences[i]) {
606- continue;
607- }
608- 
609- seq[i] = fences[i]->seq;
610- ++num_rings;
611- }
612- 
613- /* nothing to wait for ? */
614- if (num_rings == 0)
615- return -ENOENT;
616- 
617- r = radeon_fence_wait_seq_timeout(rdev, seq, intr, MAX_SCHEDULE_TIMEOUT);
618- if (r < 0) {
619- return r;
620- }
621- return 0;
622}568}
623 569 
624/**570/**
@@ -638,13 +584,16 @@ int radeon_fence_wait_next(struct radeon_device *rdev, int ring)
638 584 
639 seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL;585 seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL;
640 if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) {586 if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) {
641- /* nothing to wait for, last_seq is587+ /* nothing to wait for, last_seq is already
642- already the last emited fence */588+ * the last emited fence
589+ */
643 return -ENOENT;590 return -ENOENT;
644 }591 }
592+ 
645 r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT);593 r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT);
646 if (r < 0)594 if (r < 0)
647 return r;595 return r;
596+ 
648 return 0;597 return 0;
649}598}
650 599 
@@ -704,9 +653,8 @@ void radeon_fence_unref(struct radeon_fence **fence)
704 struct radeon_fence *tmp = *fence;653 struct radeon_fence *tmp = *fence;
705 654 
706 *fence = NULL;655 *fence = NULL;
707- if (tmp) {656+ if (tmp)
708 dma_fence_put(&tmp->base);657 dma_fence_put(&tmp->base);
709- }
710}658}
711 659 
712/**660/**
@@ -719,7 +667,7 @@ void radeon_fence_unref(struct radeon_fence **fence)
719 * Returns the number of emitted fences on the ring. Used by the667 * Returns the number of emitted fences on the ring. Used by the
720 * dynpm code to ring track activity.668 * dynpm code to ring track activity.
721 */669 */
722-unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring)670+unsigned int radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
723{671{
724 uint64_t emitted;672 uint64_t emitted;
725 673 
@@ -730,10 +678,10 @@ unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
730 emitted = rdev->fence_drv[ring].sync_seq[ring]678 emitted = rdev->fence_drv[ring].sync_seq[ring]
731 - atomic64_read(&rdev->fence_drv[ring].last_seq);679 - atomic64_read(&rdev->fence_drv[ring].last_seq);
732 /* to avoid 32bits warp around */680 /* to avoid 32bits warp around */
733- if (emitted > 0x10000000) {681+ if (emitted > 0x10000000)
734 emitted = 0x10000000;682 emitted = 0x10000000;
735- }683+ 
736- return (unsigned)emitted;684+ return (unsigned int)emitted;
737}685}
738 686 
739/**687/**
@@ -751,19 +699,16 @@ bool radeon_fence_need_sync(struct radeon_fence *fence, int dst_ring)
751{699{
752 struct radeon_fence_driver *fdrv;700 struct radeon_fence_driver *fdrv;
753 701 
754- if (!fence) {702+ if (!fence)
755 return false;703 return false;
756- }
757 704 
758- if (fence->ring == dst_ring) {705+ if (fence->ring == dst_ring)
759 return false;706 return false;
760- }
761 707 
762 /* we are protected by the ring mutex */708 /* we are protected by the ring mutex */
763 fdrv = &fence->rdev->fence_drv[dst_ring];709 fdrv = &fence->rdev->fence_drv[dst_ring];
764- if (fence->seq <= fdrv->sync_seq[fence->ring]) {710+ if (fence->seq <= fdrv->sync_seq[fence->ring])
765 return false;711 return false;
766- }
767 712 
768 return true;713 return true;
769}714}
@@ -780,23 +725,21 @@ bool radeon_fence_need_sync(struct radeon_fence *fence, int dst_ring)
780void radeon_fence_note_sync(struct radeon_fence *fence, int dst_ring)725void radeon_fence_note_sync(struct radeon_fence *fence, int dst_ring)
781{726{
782 struct radeon_fence_driver *dst, *src;727 struct radeon_fence_driver *dst, *src;
783- unsigned i;728+ unsigned int i;
784 729 
785- if (!fence) {730+ if (!fence)
786 return;731 return;
787- }
788 732 
789- if (fence->ring == dst_ring) {733+ if (fence->ring == dst_ring)
790 return;734 return;
791- }
792 735 
793 /* we are protected by the ring mutex */736 /* we are protected by the ring mutex */
794 src = &fence->rdev->fence_drv[fence->ring];737 src = &fence->rdev->fence_drv[fence->ring];
795 dst = &fence->rdev->fence_drv[dst_ring];738 dst = &fence->rdev->fence_drv[dst_ring];
796 for (i = 0; i < RADEON_NUM_RINGS; ++i) {739 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
797- if (i == dst_ring) {740+ if (i == dst_ring)
798 continue;741 continue;
799- }742+ 
800 dst->sync_seq[i] = max(dst->sync_seq[i], src->sync_seq[i]);743 dst->sync_seq[i] = max(dst->sync_seq[i], src->sync_seq[i]);
801 }744 }
802}745}
@@ -848,7 +791,7 @@ int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)
848 }791 }
849 radeon_fence_write(rdev, atomic64_read(&rdev->fence_drv[ring].last_seq), ring);792 radeon_fence_write(rdev, atomic64_read(&rdev->fence_drv[ring].last_seq), ring);
850 rdev->fence_drv[ring].initialized = true;793 rdev->fence_drv[ring].initialized = true;
851- dev_info(rdev->dev, "fence driver on ring %d use gpu addr 0x%016llx\n",794+ dev_info(rdev->dev, "fence driver on ring %d uses gpu addr 0x%016llx\n",
852 ring, rdev->fence_drv[ring].gpu_addr);795 ring, rdev->fence_drv[ring].gpu_addr);
853 return 0;796 return 0;
854}797}
@@ -895,9 +838,8 @@ void radeon_fence_driver_init(struct radeon_device *rdev)
895 int ring;838 int ring;
896 839 
897 init_waitqueue_head(&rdev->fence_queue);840 init_waitqueue_head(&rdev->fence_queue);
898- for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {841+ for (ring = 0; ring < RADEON_NUM_RINGS; ring++)
899 radeon_fence_driver_init_ring(rdev, ring);842 radeon_fence_driver_init_ring(rdev, ring);
900- }
901 843 
902 radeon_debugfs_fence_init(rdev);844 radeon_debugfs_fence_init(rdev);
903}845}
@@ -1023,6 +965,7 @@ static const char *radeon_fence_get_driver_name(struct dma_fence *fence)
1023static const char *radeon_fence_get_timeline_name(struct dma_fence *f)965static const char *radeon_fence_get_timeline_name(struct dma_fence *f)
1024{966{
1025 struct radeon_fence *fence = to_radeon_fence(f);967 struct radeon_fence *fence = to_radeon_fence(f);
968+ 
1026 switch (fence->ring) {969 switch (fence->ring) {
1027 case RADEON_RING_TYPE_GFX_INDEX: return "radeon.gfx";970 case RADEON_RING_TYPE_GFX_INDEX: return "radeon.gfx";
1028 case CAYMAN_RING_TYPE_CP1_INDEX: return "radeon.cp1";971 case CAYMAN_RING_TYPE_CP1_INDEX: return "radeon.cp1";
@@ -1032,7 +975,9 @@ static const char *radeon_fence_get_timeline_name(struct dma_fence *f)
1032 case R600_RING_TYPE_UVD_INDEX: return "radeon.uvd";975 case R600_RING_TYPE_UVD_INDEX: return "radeon.uvd";
1033 case TN_RING_TYPE_VCE1_INDEX: return "radeon.vce1";976 case TN_RING_TYPE_VCE1_INDEX: return "radeon.vce1";
1034 case TN_RING_TYPE_VCE2_INDEX: return "radeon.vce2";977 case TN_RING_TYPE_VCE2_INDEX: return "radeon.vce2";
1035- default: WARN_ON_ONCE(1); return "radeon.unk";978+ default:
979+ WARN_ON_ONCE(1);
980+ return "radeon.unk";
1036 }981 }
1037}982}
1038 983 
Mdrivers/gpu/drm/vgem/vgem_fence.c+1-1
@@ -94,7 +94,7 @@ static struct dma_fence *vgem_fence_create(struct vgem_file *vfile,
94 dma_fence_init(&fence->base, &vgem_fence_ops, &fence->lock,94 dma_fence_init(&fence->base, &vgem_fence_ops, &fence->lock,
95 dma_fence_context_alloc(1), 1);95 dma_fence_context_alloc(1), 1);
96 96 
97- timer_setup(&fence->timer, vgem_fence_timeout, 0);97+ timer_setup(&fence->timer, vgem_fence_timeout, TIMER_IRQSAFE);
98 98 
99 /* We force the fence to expire within 10s to prevent driver hangs */99 /* We force the fence to expire within 10s to prevent driver hangs */
100 mod_timer(&fence->timer, jiffies + VGEM_FENCE_TIMEOUT);100 mod_timer(&fence->timer, jiffies + VGEM_FENCE_TIMEOUT);
Mdrivers/iio/accel/bmc150-accel-core.c+5-0
@@ -569,6 +569,10 @@ static int bmc150_accel_set_interrupt(struct bmc150_accel_data *data, int i,
569 const struct bmc150_accel_interrupt_info *info = intr->info;569 const struct bmc150_accel_interrupt_info *info = intr->info;
570 int ret;570 int ret;
571 571 
572+ /* We do not always have an IRQ */
573+ if (data->irq <= 0)
574+ return 0;
575+ 
572 if (state) {576 if (state) {
573 if (atomic_inc_return(&intr->users) > 1)577 if (atomic_inc_return(&intr->users) > 1)
574 return 0;578 return 0;
@@ -1743,6 +1747,7 @@ int bmc150_accel_core_probe(struct device *dev, struct regmap *regmap, int irq,
1743 }1747 }
1744 1748 
1745 if (irq > 0) {1749 if (irq > 0) {
1750+ data->irq = irq;
1746 ret = devm_request_threaded_irq(dev, irq,1751 ret = devm_request_threaded_irq(dev, irq,
1747 bmc150_accel_irq_handler,1752 bmc150_accel_irq_handler,
1748 bmc150_accel_irq_thread_handler,1753 bmc150_accel_irq_thread_handler,
Mdrivers/iio/accel/bmc150-accel.h+1-0
@@ -57,6 +57,7 @@ enum bmc150_accel_trigger_id {
57 57 
58struct bmc150_accel_data {58struct bmc150_accel_data {
59 struct regmap *regmap;59 struct regmap *regmap;
60+ int irq;
60 struct regulator_bulk_data regulators[2];61 struct regulator_bulk_data regulators[2];
61 struct bmc150_accel_interrupt interrupts[BMC150_ACCEL_INTERRUPTS];62 struct bmc150_accel_interrupt interrupts[BMC150_ACCEL_INTERRUPTS];
62 struct bmc150_accel_trigger triggers[BMC150_ACCEL_TRIGGERS];63 struct bmc150_accel_trigger triggers[BMC150_ACCEL_TRIGGERS];
Mdrivers/infiniband/core/addr.c+10-23
@@ -80,37 +80,25 @@ static const struct nla_policy ib_nl_addr_policy[LS_NLA_TYPE_MAX] = {
80 .min = sizeof(struct rdma_nla_ls_gid)},80 .min = sizeof(struct rdma_nla_ls_gid)},
81};81};
82 82 
83-static inline bool ib_nl_is_good_ip_resp(const struct nlmsghdr *nlh)83+static void ib_nl_process_ip_rsep(const struct nlmsghdr *nlh)
84{84{
85 struct nlattr *tb[LS_NLA_TYPE_MAX] = {};85 struct nlattr *tb[LS_NLA_TYPE_MAX] = {};
86+ union ib_gid gid;
87+ struct addr_req *req;
88+ int found = 0;
86 int ret;89 int ret;
87 90 
88 if (nlh->nlmsg_flags & RDMA_NL_LS_F_ERR)91 if (nlh->nlmsg_flags & RDMA_NL_LS_F_ERR)
89- return false;92+ return;
90 93 
91 ret = nla_parse_deprecated(tb, LS_NLA_TYPE_MAX - 1, nlmsg_data(nlh),94 ret = nla_parse_deprecated(tb, LS_NLA_TYPE_MAX - 1, nlmsg_data(nlh),
92 nlmsg_len(nlh), ib_nl_addr_policy, NULL);95 nlmsg_len(nlh), ib_nl_addr_policy, NULL);
93 if (ret)96 if (ret)
94- return false;97+ return;
95 98 
96- return true;99+ if (!tb[LS_NLA_TYPE_DGID])
97-}100+ return;
98- 101+ memcpy(&gid, nla_data(tb[LS_NLA_TYPE_DGID]), sizeof(gid));
99-static void ib_nl_process_good_ip_rsep(const struct nlmsghdr *nlh)
100-{
101- const struct nlattr *head, *curr;
102- union ib_gid gid;
103- struct addr_req *req;
104- int len, rem;
105- int found = 0;
106- 
107- head = (const struct nlattr *)nlmsg_data(nlh);
108- len = nlmsg_len(nlh);
109- 
110- nla_for_each_attr(curr, head, len, rem) {
111- if (curr->nla_type == LS_NLA_TYPE_DGID)
112- memcpy(&gid, nla_data(curr), nla_len(curr));
113- }
114 102 
115 spin_lock_bh(&lock);103 spin_lock_bh(&lock);
116 list_for_each_entry(req, &req_list, list) {104 list_for_each_entry(req, &req_list, list) {
@@ -137,8 +125,7 @@ int ib_nl_handle_ip_res_resp(struct sk_buff *skb,
137 !(NETLINK_CB(skb).sk))125 !(NETLINK_CB(skb).sk))
138 return -EPERM;126 return -EPERM;
139 127 
140- if (ib_nl_is_good_ip_resp(nlh))128+ ib_nl_process_ip_rsep(nlh);
141- ib_nl_process_good_ip_rsep(nlh);
142 129 
143 return 0;130 return 0;
144}131}
Mdrivers/input/keyboard/lkkbd.c+4-1
@@ -670,7 +670,8 @@ static int lkkbd_connect(struct serio *serio, struct serio_driver *drv)
670 670 
671 return 0;671 return 0;
672 672 
673- fail3: serio_close(serio);673+ fail3: disable_work_sync(&lk->tq);
674+ serio_close(serio);
674 fail2: serio_set_drvdata(serio, NULL);675 fail2: serio_set_drvdata(serio, NULL);
675 fail1: input_free_device(input_dev);676 fail1: input_free_device(input_dev);
676 kfree(lk);677 kfree(lk);
@@ -684,6 +685,8 @@ static void lkkbd_disconnect(struct serio *serio)
684{685{
685 struct lkkbd *lk = serio_get_drvdata(serio);686 struct lkkbd *lk = serio_get_drvdata(serio);
686 687 
688+ disable_work_sync(&lk->tq);
689+ 
687 input_get_device(lk->dev);690 input_get_device(lk->dev);
688 input_unregister_device(lk->dev);691 input_unregister_device(lk->dev);
689 serio_close(serio);692 serio_close(serio);
Mdrivers/irqchip/irq-mchp-eic.c+1-1
@@ -166,7 +166,7 @@ static int mchp_eic_domain_alloc(struct irq_domain *domain, unsigned int virq,
166 166 
167 ret = irq_domain_translate_twocell(domain, fwspec, &hwirq, &type);167 ret = irq_domain_translate_twocell(domain, fwspec, &hwirq, &type);
168 if (ret || hwirq >= MCHP_EIC_NIRQ)168 if (ret || hwirq >= MCHP_EIC_NIRQ)
169- return ret;169+ return ret ?: -EINVAL;
170 170 
171 switch (type) {171 switch (type) {
172 case IRQ_TYPE_EDGE_RISING:172 case IRQ_TYPE_EDGE_RISING:
Mdrivers/net/ethernet/emulex/benet/be_cmds.c+1-1
@@ -3852,8 +3852,8 @@ int be_cmd_set_mac_list(struct be_adapter *adapter, u8 *mac_array,
3852 status = be_mcc_notify_wait(adapter);3852 status = be_mcc_notify_wait(adapter);
3853 3853 
3854err:3854err:
3855- dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3856 spin_unlock_bh(&adapter->mcc_lock);3855 spin_unlock_bh(&adapter->mcc_lock);
3856+ dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3857 return status;3857 return status;
3858}3858}
3859 3859 
Mdrivers/net/ethernet/stmicro/stmmac/stmmac_main.c+2-1
@@ -5522,7 +5522,8 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
5522 stmmac_rx_vlan(priv->dev, skb);5522 stmmac_rx_vlan(priv->dev, skb);
5523 skb->protocol = eth_type_trans(skb, priv->dev);5523 skb->protocol = eth_type_trans(skb, priv->dev);
5524 5524 
5525- if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb))5525+ if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb) ||
5526+ (status & csum_none))
5526 skb_checksum_none_assert(skb);5527 skb_checksum_none_assert(skb);
5527 else5528 else
5528 skb->ip_summed = CHECKSUM_UNNECESSARY;5529 skb->ip_summed = CHECKSUM_UNNECESSARY;
Mdrivers/net/ethernet/ti/netcp_core.c+5-5
@@ -1338,10 +1338,10 @@ int netcp_txpipe_open(struct netcp_tx_pipe *tx_pipe)
1338 1338 
1339 tx_pipe->dma_channel = knav_dma_open_channel(dev,1339 tx_pipe->dma_channel = knav_dma_open_channel(dev,
1340 tx_pipe->dma_chan_name, &config);1340 tx_pipe->dma_chan_name, &config);
1341- if (IS_ERR(tx_pipe->dma_channel)) {1341+ if (!tx_pipe->dma_channel) {
1342 dev_err(dev, "failed opening tx chan(%s)\n",1342 dev_err(dev, "failed opening tx chan(%s)\n",
1343 tx_pipe->dma_chan_name);1343 tx_pipe->dma_chan_name);
1344- ret = PTR_ERR(tx_pipe->dma_channel);1344+ ret = -EINVAL;
1345 goto err;1345 goto err;
1346 }1346 }
1347 1347 
@@ -1359,7 +1359,7 @@ int netcp_txpipe_open(struct netcp_tx_pipe *tx_pipe)
1359 return 0;1359 return 0;
1360 1360 
1361err:1361err:
1362- if (!IS_ERR_OR_NULL(tx_pipe->dma_channel))1362+ if (tx_pipe->dma_channel)
1363 knav_dma_close_channel(tx_pipe->dma_channel);1363 knav_dma_close_channel(tx_pipe->dma_channel);
1364 tx_pipe->dma_channel = NULL;1364 tx_pipe->dma_channel = NULL;
1365 return ret;1365 return ret;
@@ -1678,10 +1678,10 @@ static int netcp_setup_navigator_resources(struct net_device *ndev)
1678 1678 
1679 netcp->rx_channel = knav_dma_open_channel(netcp->netcp_device->device,1679 netcp->rx_channel = knav_dma_open_channel(netcp->netcp_device->device,
1680 netcp->dma_chan_name, &config);1680 netcp->dma_chan_name, &config);
1681- if (IS_ERR(netcp->rx_channel)) {1681+ if (!netcp->rx_channel) {
1682 dev_err(netcp->ndev_dev, "failed opening rx chan(%s\n",1682 dev_err(netcp->ndev_dev, "failed opening rx chan(%s\n",
1683 netcp->dma_chan_name);1683 netcp->dma_chan_name);
1684- ret = PTR_ERR(netcp->rx_channel);1684+ ret = -EINVAL;
1685 goto fail;1685 goto fail;
1686 }1686 }
1687 1687 
Mdrivers/net/wireless/mediatek/mt76/mt7615/mcu.c+3-1
@@ -870,8 +870,10 @@ mt7615_mcu_wtbl_sta_add(struct mt7615_phy *phy, struct ieee80211_vif *vif,
870 wtbl_hdr = mt76_connac_mcu_alloc_wtbl_req(&dev->mt76, &msta->wcid,870 wtbl_hdr = mt76_connac_mcu_alloc_wtbl_req(&dev->mt76, &msta->wcid,
871 WTBL_RESET_AND_SET, NULL,871 WTBL_RESET_AND_SET, NULL,
872 &wskb);872 &wskb);
873- if (IS_ERR(wtbl_hdr))873+ if (IS_ERR(wtbl_hdr)) {
874+ dev_kfree_skb(sskb);
874 return PTR_ERR(wtbl_hdr);875 return PTR_ERR(wtbl_hdr);
876+ }
875 877 
876 if (enable) {878 if (enable) {
877 mt76_connac_mcu_wtbl_generic_tlv(&dev->mt76, wskb, vif, sta,879 mt76_connac_mcu_wtbl_generic_tlv(&dev->mt76, wskb, vif, sta,
Mdrivers/net/wireless/realtek/rtl818x/rtl8180/dev.c+2-7
@@ -1023,9 +1023,6 @@ static int rtl8180_init_rx_ring(struct ieee80211_hw *dev)
1023 dma_addr_t *mapping;1023 dma_addr_t *mapping;
1024 entry = priv->rx_ring + priv->rx_ring_sz*i;1024 entry = priv->rx_ring + priv->rx_ring_sz*i;
1025 if (!skb) {1025 if (!skb) {
1026- dma_free_coherent(&priv->pdev->dev,
1027- priv->rx_ring_sz * 32,
1028- priv->rx_ring, priv->rx_ring_dma);
1029 wiphy_err(dev->wiphy, "Cannot allocate RX skb\n");1026 wiphy_err(dev->wiphy, "Cannot allocate RX skb\n");
1030 return -ENOMEM;1027 return -ENOMEM;
1031 }1028 }
@@ -1037,9 +1034,7 @@ static int rtl8180_init_rx_ring(struct ieee80211_hw *dev)
1037 1034 
1038 if (dma_mapping_error(&priv->pdev->dev, *mapping)) {1035 if (dma_mapping_error(&priv->pdev->dev, *mapping)) {
1039 kfree_skb(skb);1036 kfree_skb(skb);
1040- dma_free_coherent(&priv->pdev->dev,1037+ priv->rx_buf[i] = NULL;
1041- priv->rx_ring_sz * 32,
1042- priv->rx_ring, priv->rx_ring_dma);
1043 wiphy_err(dev->wiphy, "Cannot map DMA for RX skb\n");1038 wiphy_err(dev->wiphy, "Cannot map DMA for RX skb\n");
1044 return -ENOMEM;1039 return -ENOMEM;
1045 }1040 }
@@ -1130,7 +1125,7 @@ static int rtl8180_start(struct ieee80211_hw *dev)
1130 1125 
1131 ret = rtl8180_init_rx_ring(dev);1126 ret = rtl8180_init_rx_ring(dev);
1132 if (ret)1127 if (ret)
1133- return ret;1128+ goto err_free_rings;
1134 1129 
1135 for (i = 0; i < (dev->queues + 1); i++)1130 for (i = 0; i < (dev->queues + 1); i++)
1136 if ((ret = rtl8180_init_tx_ring(dev, i, 16)))1131 if ((ret = rtl8180_init_tx_ring(dev, i, 16)))
Mdrivers/net/wireless/realtek/rtl818x/rtl8187/dev.c+19-8
@@ -338,14 +338,16 @@ static void rtl8187_rx_cb(struct urb *urb)
338 spin_unlock_irqrestore(&priv->rx_queue.lock, f);338 spin_unlock_irqrestore(&priv->rx_queue.lock, f);
339 skb_put(skb, urb->actual_length);339 skb_put(skb, urb->actual_length);
340 340 
341- if (unlikely(urb->status)) {341+ if (unlikely(urb->status))
342- dev_kfree_skb_irq(skb);342+ goto free_skb;
343- return;
344- }
345 343 
346 if (!priv->is_rtl8187b) {344 if (!priv->is_rtl8187b) {
347- struct rtl8187_rx_hdr *hdr =345+ struct rtl8187_rx_hdr *hdr;
348- (typeof(hdr))(skb_tail_pointer(skb) - sizeof(*hdr));346+ 
347+ if (skb->len < sizeof(struct rtl8187_rx_hdr))
348+ goto free_skb;
349+ 
350+ hdr = (typeof(hdr))(skb_tail_pointer(skb) - sizeof(*hdr));
349 flags = le32_to_cpu(hdr->flags);351 flags = le32_to_cpu(hdr->flags);
350 /* As with the RTL8187B below, the AGC is used to calculate352 /* As with the RTL8187B below, the AGC is used to calculate
351 * signal strength. In this case, the scaling353 * signal strength. In this case, the scaling
@@ -355,8 +357,12 @@ static void rtl8187_rx_cb(struct urb *urb)
355 rx_status.antenna = (hdr->signal >> 7) & 1;357 rx_status.antenna = (hdr->signal >> 7) & 1;
356 rx_status.mactime = le64_to_cpu(hdr->mac_time);358 rx_status.mactime = le64_to_cpu(hdr->mac_time);
357 } else {359 } else {
358- struct rtl8187b_rx_hdr *hdr =360+ struct rtl8187b_rx_hdr *hdr;
359- (typeof(hdr))(skb_tail_pointer(skb) - sizeof(*hdr));361+ 
362+ if (skb->len < sizeof(struct rtl8187b_rx_hdr))
363+ goto free_skb;
364+ 
365+ hdr = (typeof(hdr))(skb_tail_pointer(skb) - sizeof(*hdr));
360 /* The Realtek datasheet for the RTL8187B shows that the RX366 /* The Realtek datasheet for the RTL8187B shows that the RX
361 * header contains the following quantities: signal quality,367 * header contains the following quantities: signal quality,
362 * RSSI, AGC, the received power in dB, and the measured SNR.368 * RSSI, AGC, the received power in dB, and the measured SNR.
@@ -409,6 +415,11 @@ static void rtl8187_rx_cb(struct urb *urb)
409 skb_unlink(skb, &priv->rx_queue);415 skb_unlink(skb, &priv->rx_queue);
410 dev_kfree_skb_irq(skb);416 dev_kfree_skb_irq(skb);
411 }417 }
418+ return;
419+ 
420+free_skb:
421+ dev_kfree_skb_irq(skb);
422+ return;
412}423}
413 424 
414static int rtl8187_init_urbs(struct ieee80211_hw *dev)425static int rtl8187_init_urbs(struct ieee80211_hw *dev)
Mdrivers/platform/x86/hp/hp-bioscfg/enum-attributes.c+2-2
@@ -210,7 +210,7 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum
210 case PREREQUISITES:210 case PREREQUISITES:
211 size = min_t(u32, enum_data->common.prerequisites_size, MAX_PREREQUISITES_SIZE);211 size = min_t(u32, enum_data->common.prerequisites_size, MAX_PREREQUISITES_SIZE);
212 for (reqs = 0; reqs < size; reqs++) {212 for (reqs = 0; reqs < size; reqs++) {
213- if (elem >= enum_obj_count) {213+ if (elem + reqs >= enum_obj_count) {
214 pr_err("Error enum-objects package is too small\n");214 pr_err("Error enum-objects package is too small\n");
215 return -EINVAL;215 return -EINVAL;
216 }216 }
@@ -261,7 +261,7 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum
261 261 
262 for (pos_values = 0; pos_values < size && pos_values < MAX_VALUES_SIZE;262 for (pos_values = 0; pos_values < size && pos_values < MAX_VALUES_SIZE;
263 pos_values++) {263 pos_values++) {
264- if (elem >= enum_obj_count) {264+ if (elem + pos_values >= enum_obj_count) {
265 pr_err("Error enum-objects package is too small\n");265 pr_err("Error enum-objects package is too small\n");
266 return -EINVAL;266 return -EINVAL;
267 }267 }
Mdrivers/platform/x86/hp/hp-bioscfg/int-attributes.c+1-1
@@ -228,7 +228,7 @@ static int hp_populate_integer_elements_from_package(union acpi_object *integer_
228 size = min_t(u32, integer_data->common.prerequisites_size, MAX_PREREQUISITES_SIZE);228 size = min_t(u32, integer_data->common.prerequisites_size, MAX_PREREQUISITES_SIZE);
229 229 
230 for (reqs = 0; reqs < size; reqs++) {230 for (reqs = 0; reqs < size; reqs++) {
231- if (elem >= integer_obj_count) {231+ if (elem + reqs >= integer_obj_count) {
232 pr_err("Error elem-objects package is too small\n");232 pr_err("Error elem-objects package is too small\n");
233 return -EINVAL;233 return -EINVAL;
234 }234 }
Mdrivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c+5-0
@@ -220,6 +220,11 @@ static int hp_populate_ordered_list_elements_from_package(union acpi_object *ord
220 size = min_t(u32, ordered_list_data->common.prerequisites_size,220 size = min_t(u32, ordered_list_data->common.prerequisites_size,
221 MAX_PREREQUISITES_SIZE);221 MAX_PREREQUISITES_SIZE);
222 for (reqs = 0; reqs < size; reqs++) {222 for (reqs = 0; reqs < size; reqs++) {
223+ if (elem + reqs >= order_obj_count) {
224+ pr_err("Error elem-objects package is too small\n");
225+ return -EINVAL;
226+ }
227+ 
223 ret = hp_convert_hexstr_to_str(order_obj[elem + reqs].string.pointer,228 ret = hp_convert_hexstr_to_str(order_obj[elem + reqs].string.pointer,
224 order_obj[elem + reqs].string.length,229 order_obj[elem + reqs].string.length,
225 &str_value, &value_len);230 &str_value, &value_len);
Mdrivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c+5-0
@@ -309,6 +309,11 @@ static int hp_populate_password_elements_from_package(union acpi_object *passwor
309 MAX_PREREQUISITES_SIZE);309 MAX_PREREQUISITES_SIZE);
310 310 
311 for (reqs = 0; reqs < size; reqs++) {311 for (reqs = 0; reqs < size; reqs++) {
312+ if (elem + reqs >= password_obj_count) {
313+ pr_err("Error elem-objects package is too small\n");
314+ return -EINVAL;
315+ }
316+ 
312 ret = hp_convert_hexstr_to_str(password_obj[elem + reqs].string.pointer,317 ret = hp_convert_hexstr_to_str(password_obj[elem + reqs].string.pointer,
313 password_obj[elem + reqs].string.length,318 password_obj[elem + reqs].string.length,
314 &str_value, &value_len);319 &str_value, &value_len);
Mdrivers/platform/x86/hp/hp-bioscfg/string-attributes.c+1-1
@@ -219,7 +219,7 @@ static int hp_populate_string_elements_from_package(union acpi_object *string_ob
219 MAX_PREREQUISITES_SIZE);219 MAX_PREREQUISITES_SIZE);
220 220 
221 for (reqs = 0; reqs < size; reqs++) {221 for (reqs = 0; reqs < size; reqs++) {
222- if (elem >= string_obj_count) {222+ if (elem + reqs >= string_obj_count) {
223 pr_err("Error elem-objects package is too small\n");223 pr_err("Error elem-objects package is too small\n");
224 return -EINVAL;224 return -EINVAL;
225 }225 }
Mdrivers/scsi/qla2xxx/qla_nvme.c+1-1
@@ -1293,7 +1293,7 @@ void qla2xxx_process_purls_iocb(void **pkt, struct rsp_que **rsp)
1293 a.reason = FCNVME_RJT_RC_LOGIC;1293 a.reason = FCNVME_RJT_RC_LOGIC;
1294 a.explanation = FCNVME_RJT_EXP_NONE;1294 a.explanation = FCNVME_RJT_EXP_NONE;
1295 xmt_reject = true;1295 xmt_reject = true;
1296- kfree(item);1296+ qla24xx_free_purex_item(item);
1297 goto out;1297 goto out;
1298 }1298 }
1299 1299 
Mdrivers/soc/ti/knav_dma.c+7-7
@@ -402,7 +402,7 @@ static int of_channel_match_helper(struct device_node *np, const char *name,
402 * @name: slave channel name402 * @name: slave channel name
403 * @config: dma configuration parameters403 * @config: dma configuration parameters
404 *404 *
405- * Returns pointer to appropriate DMA channel on success or error.405+ * Return: Pointer to appropriate DMA channel on success or NULL on error.
406 */406 */
407void *knav_dma_open_channel(struct device *dev, const char *name,407void *knav_dma_open_channel(struct device *dev, const char *name,
408 struct knav_dma_cfg *config)408 struct knav_dma_cfg *config)
@@ -414,13 +414,13 @@ void *knav_dma_open_channel(struct device *dev, const char *name,
414 414 
415 if (!kdev) {415 if (!kdev) {
416 pr_err("keystone-navigator-dma driver not registered\n");416 pr_err("keystone-navigator-dma driver not registered\n");
417- return (void *)-EINVAL;417+ return NULL;
418 }418 }
419 419 
420 chan_num = of_channel_match_helper(dev->of_node, name, &instance);420 chan_num = of_channel_match_helper(dev->of_node, name, &instance);
421 if (chan_num < 0) {421 if (chan_num < 0) {
422 dev_err(kdev->dev, "No DMA instance with name %s\n", name);422 dev_err(kdev->dev, "No DMA instance with name %s\n", name);
423- return (void *)-EINVAL;423+ return NULL;
424 }424 }
425 425 
426 dev_dbg(kdev->dev, "initializing %s channel %d from DMA %s\n",426 dev_dbg(kdev->dev, "initializing %s channel %d from DMA %s\n",
@@ -431,7 +431,7 @@ void *knav_dma_open_channel(struct device *dev, const char *name,
431 if (config->direction != DMA_MEM_TO_DEV &&431 if (config->direction != DMA_MEM_TO_DEV &&
432 config->direction != DMA_DEV_TO_MEM) {432 config->direction != DMA_DEV_TO_MEM) {
433 dev_err(kdev->dev, "bad direction\n");433 dev_err(kdev->dev, "bad direction\n");
434- return (void *)-EINVAL;434+ return NULL;
435 }435 }
436 436 
437 /* Look for correct dma instance */437 /* Look for correct dma instance */
@@ -443,7 +443,7 @@ void *knav_dma_open_channel(struct device *dev, const char *name,
443 }443 }
444 if (!dma) {444 if (!dma) {
445 dev_err(kdev->dev, "No DMA instance with name %s\n", instance);445 dev_err(kdev->dev, "No DMA instance with name %s\n", instance);
446- return (void *)-EINVAL;446+ return NULL;
447 }447 }
448 448 
449 /* Look for correct dma channel from dma instance */449 /* Look for correct dma channel from dma instance */
@@ -463,14 +463,14 @@ void *knav_dma_open_channel(struct device *dev, const char *name,
463 if (!chan) {463 if (!chan) {
464 dev_err(kdev->dev, "channel %d is not in DMA %s\n",464 dev_err(kdev->dev, "channel %d is not in DMA %s\n",
465 chan_num, instance);465 chan_num, instance);
466- return (void *)-EINVAL;466+ return NULL;
467 }467 }
468 468 
469 if (atomic_read(&chan->ref_count) >= 1) {469 if (atomic_read(&chan->ref_count) >= 1) {
470 if (!check_config(chan, config)) {470 if (!check_config(chan, config)) {
471 dev_err(kdev->dev, "channel %d config miss-match\n",471 dev_err(kdev->dev, "channel %d config miss-match\n",
472 chan_num);472 chan_num);
473- return (void *)-EINVAL;473+ return NULL;
474 }474 }
475 }475 }
476 476 
Mdrivers/spi/spi-tegra210-quad.c+20-2
@@ -999,8 +999,10 @@ static void tegra_qspi_handle_error(struct tegra_qspi *tqspi)
999 dev_err(tqspi->dev, "error in transfer, fifo status 0x%08x\n", tqspi->status_reg);999 dev_err(tqspi->dev, "error in transfer, fifo status 0x%08x\n", tqspi->status_reg);
1000 tegra_qspi_dump_regs(tqspi);1000 tegra_qspi_dump_regs(tqspi);
1001 tegra_qspi_flush_fifos(tqspi, true);1001 tegra_qspi_flush_fifos(tqspi, true);
1002- if (device_reset(tqspi->dev) < 0)1002+ if (device_reset(tqspi->dev) < 0) {
1003 dev_warn_once(tqspi->dev, "device reset failed\n");1003 dev_warn_once(tqspi->dev, "device reset failed\n");
1004+ tegra_qspi_mask_clear_irq(tqspi);
1005+ }
1004}1006}
1005 1007 
1006static void tegra_qspi_transfer_end(struct spi_device *spi)1008static void tegra_qspi_transfer_end(struct spi_device *spi)
@@ -1145,9 +1147,11 @@ static int tegra_qspi_combined_seq_xfer(struct tegra_qspi *tqspi,
1145 }1147 }
1146 1148 
1147 /* Reset controller if timeout happens */1149 /* Reset controller if timeout happens */
1148- if (device_reset(tqspi->dev) < 0)1150+ if (device_reset(tqspi->dev) < 0) {
1149 dev_warn_once(tqspi->dev,1151 dev_warn_once(tqspi->dev,
1150 "device reset failed\n");1152 "device reset failed\n");
1153+ tegra_qspi_mask_clear_irq(tqspi);
1154+ }
1151 ret = -EIO;1155 ret = -EIO;
1152 goto exit;1156 goto exit;
1153 }1157 }
@@ -1169,11 +1173,13 @@ static int tegra_qspi_combined_seq_xfer(struct tegra_qspi *tqspi,
1169 tegra_qspi_transfer_end(spi);1173 tegra_qspi_transfer_end(spi);
1170 spi_transfer_delay_exec(xfer);1174 spi_transfer_delay_exec(xfer);
1171 }1175 }
1176+ tqspi->curr_xfer = NULL;
1172 transfer_phase++;1177 transfer_phase++;
1173 }1178 }
1174 ret = 0;1179 ret = 0;
1175 1180 
1176exit:1181exit:
1182+ tqspi->curr_xfer = NULL;
1177 msg->status = ret;1183 msg->status = ret;
1178 1184 
1179 return ret;1185 return ret;
@@ -1257,6 +1263,8 @@ static int tegra_qspi_non_combined_seq_xfer(struct tegra_qspi *tqspi,
1257 msg->actual_length += xfer->len + dummy_bytes;1263 msg->actual_length += xfer->len + dummy_bytes;
1258 1264 
1259complete_xfer:1265complete_xfer:
1266+ tqspi->curr_xfer = NULL;
1267+ 
1260 if (ret < 0) {1268 if (ret < 0) {
1261 tegra_qspi_transfer_end(spi);1269 tegra_qspi_transfer_end(spi);
1262 spi_transfer_delay_exec(xfer);1270 spi_transfer_delay_exec(xfer);
@@ -1353,6 +1361,7 @@ static irqreturn_t handle_cpu_based_xfer(struct tegra_qspi *tqspi)
1353 tegra_qspi_calculate_curr_xfer_param(tqspi, t);1361 tegra_qspi_calculate_curr_xfer_param(tqspi, t);
1354 tegra_qspi_start_cpu_based_transfer(tqspi, t);1362 tegra_qspi_start_cpu_based_transfer(tqspi, t);
1355exit:1363exit:
1364+ tqspi->curr_xfer = NULL;
1356 spin_unlock_irqrestore(&tqspi->lock, flags);1365 spin_unlock_irqrestore(&tqspi->lock, flags);
1357 return IRQ_HANDLED;1366 return IRQ_HANDLED;
1358}1367}
@@ -1436,6 +1445,15 @@ static irqreturn_t tegra_qspi_isr_thread(int irq, void *context_data)
1436{1445{
1437 struct tegra_qspi *tqspi = context_data;1446 struct tegra_qspi *tqspi = context_data;
1438 1447 
1448+ /*
1449+ * Occasionally the IRQ thread takes a long time to wake up (usually
1450+ * when the CPU that it's running on is excessively busy) and we have
1451+ * already reached the timeout before and cleaned up the timed out
1452+ * transfer. Avoid any processing in that case and bail out early.
1453+ */
1454+ if (!tqspi->curr_xfer)
1455+ return IRQ_NONE;
1456+ 
1439 tqspi->status_reg = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS);1457 tqspi->status_reg = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS);
1440 1458 
1441 if (tqspi->cur_direction & DATA_DIR_TX)1459 if (tqspi->cur_direction & DATA_DIR_TX)
Mdrivers/staging/most/Kconfig+0-2
@@ -24,6 +24,4 @@ source "drivers/staging/most/video/Kconfig"
24 24 
25source "drivers/staging/most/dim2/Kconfig"25source "drivers/staging/most/dim2/Kconfig"
26 26 
27-source "drivers/staging/most/i2c/Kconfig"
28- 
29endif27endif
Mdrivers/staging/most/Makefile+0-1
@@ -3,4 +3,3 @@
3obj-$(CONFIG_MOST_NET) += net/3obj-$(CONFIG_MOST_NET) += net/
4obj-$(CONFIG_MOST_VIDEO) += video/4obj-$(CONFIG_MOST_VIDEO) += video/
5obj-$(CONFIG_MOST_DIM2) += dim2/5obj-$(CONFIG_MOST_DIM2) += dim2/
6-obj-$(CONFIG_MOST_I2C) += i2c/
Ddrivers/staging/most/i2c/Kconfig+0-13
@@ -1,13 +0,0 @@
1-# SPDX-License-Identifier: GPL-2.0
2-#
3-# MOST I2C configuration
4-#
5- 
6-config MOST_I2C
7- tristate "I2C"
8- depends on I2C
9- help
10- Say Y here if you want to connect via I2C to network transceiver.
11- 
12- To compile this driver as a module, choose M here: the
13- module will be called most_i2c.
Ddrivers/staging/most/i2c/Makefile+0-4
@@ -1,4 +0,0 @@
1-# SPDX-License-Identifier: GPL-2.0
2-obj-$(CONFIG_MOST_I2C) += most_i2c.o
3- 
4-most_i2c-objs := i2c.o
Ddrivers/staging/most/i2c/i2c.c+0-374
@@ -1,374 +0,0 @@
1-// SPDX-License-Identifier: GPL-2.0
2-/*
3- * i2c.c - Hardware Dependent Module for I2C Interface
4- *
5- * Copyright (C) 2013-2015, Microchip Technology Germany II GmbH & Co. KG
6- */
7- 
8-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9- 
10-#include <linux/init.h>
11-#include <linux/module.h>
12-#include <linux/slab.h>
13-#include <linux/i2c.h>
14-#include <linux/interrupt.h>
15-#include <linux/err.h>
16-#include <linux/most.h>
17- 
18-enum { CH_RX, CH_TX, NUM_CHANNELS };
19- 
20-#define MAX_BUFFERS_CONTROL 32
21-#define MAX_BUF_SIZE_CONTROL 256
22- 
23-/**
24- * list_first_mbo - get the first mbo from a list
25- * @ptr: the list head to take the mbo from.
26- */
27-#define list_first_mbo(ptr) \
28- list_first_entry(ptr, struct mbo, list)
29- 
30-static unsigned int polling_rate;
31-module_param(polling_rate, uint, 0644);
32-MODULE_PARM_DESC(polling_rate, "Polling rate [Hz]. Default = 0 (use IRQ)");
33- 
34-struct hdm_i2c {
35- struct most_interface most_iface;
36- struct most_channel_capability capabilities[NUM_CHANNELS];
37- struct i2c_client *client;
38- struct rx {
39- struct delayed_work dwork;
40- struct list_head list;
41- bool int_disabled;
42- unsigned int delay;
43- } rx;
44- char name[64];
45-};
46- 
47-static inline struct hdm_i2c *to_hdm(struct most_interface *iface)
48-{
49- return container_of(iface, struct hdm_i2c, most_iface);
50-}
51- 
52-static irqreturn_t most_irq_handler(int, void *);
53-static void pending_rx_work(struct work_struct *);
54- 
55-/**
56- * configure_channel - called from MOST core to configure a channel
57- * @most_iface: interface the channel belongs to
58- * @ch_idx: channel to be configured
59- * @channel_config: structure that holds the configuration information
60- *
61- * Return 0 on success, negative on failure.
62- *
63- * Receives configuration information from MOST core and initialize the
64- * corresponding channel.
65- */
66-static int configure_channel(struct most_interface *most_iface,
67- int ch_idx,
68- struct most_channel_config *channel_config)
69-{
70- int ret;
71- struct hdm_i2c *dev = to_hdm(most_iface);
72- unsigned int delay, pr;
73- 
74- BUG_ON(ch_idx < 0 || ch_idx >= NUM_CHANNELS);
75- 
76- if (channel_config->data_type != MOST_CH_CONTROL) {
77- pr_err("bad data type for channel %d\n", ch_idx);
78- return -EPERM;
79- }
80- 
81- if (channel_config->direction != dev->capabilities[ch_idx].direction) {
82- pr_err("bad direction for channel %d\n", ch_idx);
83- return -EPERM;
84- }
85- 
86- if (channel_config->direction == MOST_CH_RX) {
87- if (!polling_rate) {
88- if (dev->client->irq <= 0) {
89- pr_err("bad irq: %d\n", dev->client->irq);
90- return -ENOENT;
91- }
92- dev->rx.int_disabled = false;
93- ret = request_irq(dev->client->irq, most_irq_handler, 0,
94- dev->client->name, dev);
95- if (ret) {
96- pr_err("request_irq(%d) failed: %d\n",
97- dev->client->irq, ret);
98- return ret;
99- }
100- } else {
101- delay = msecs_to_jiffies(MSEC_PER_SEC / polling_rate);
102- dev->rx.delay = delay ? delay : 1;
103- pr = MSEC_PER_SEC / jiffies_to_msecs(dev->rx.delay);
104- pr_info("polling rate is %u Hz\n", pr);
105- }
106- }
107- 
108- return 0;
109-}
110- 
111-/**
112- * enqueue - called from MOST core to enqueue a buffer for data transfer
113- * @most_iface: intended interface
114- * @ch_idx: ID of the channel the buffer is intended for
115- * @mbo: pointer to the buffer object
116- *
117- * Return 0 on success, negative on failure.
118- *
119- * Transmit the data over I2C if it is a "write" request or push the buffer into
120- * list if it is an "read" request
121- */
122-static int enqueue(struct most_interface *most_iface,
123- int ch_idx, struct mbo *mbo)
124-{
125- struct hdm_i2c *dev = to_hdm(most_iface);
126- int ret;
127- 
128- BUG_ON(ch_idx < 0 || ch_idx >= NUM_CHANNELS);
129- 
130- if (ch_idx == CH_RX) {
131- /* RX */
132- if (!polling_rate)
133- disable_irq(dev->client->irq);
134- cancel_delayed_work_sync(&dev->rx.dwork);
135- list_add_tail(&mbo->list, &dev->rx.list);
136- if (dev->rx.int_disabled || polling_rate)
137- pending_rx_work(&dev->rx.dwork.work);
138- if (!polling_rate)
139- enable_irq(dev->client->irq);
140- } else {
141- /* TX */
142- ret = i2c_master_send(dev->client, mbo->virt_address,
143- mbo->buffer_length);
144- if (ret <= 0) {
145- mbo->processed_length = 0;
146- mbo->status = MBO_E_INVAL;
147- } else {
148- mbo->processed_length = mbo->buffer_length;
149- mbo->status = MBO_SUCCESS;
150- }
151- mbo->complete(mbo);
152- }
153- 
154- return 0;
155-}
156- 
157-/**
158- * poison_channel - called from MOST core to poison buffers of a channel
159- * @most_iface: pointer to the interface the channel to be poisoned belongs to
160- * @ch_idx: corresponding channel ID
161- *
162- * Return 0 on success, negative on failure.
163- *
164- * If channel direction is RX, complete the buffers in list with
165- * status MBO_E_CLOSE
166- */
167-static int poison_channel(struct most_interface *most_iface,
168- int ch_idx)
169-{
170- struct hdm_i2c *dev = to_hdm(most_iface);
171- struct mbo *mbo;
172- 
173- BUG_ON(ch_idx < 0 || ch_idx >= NUM_CHANNELS);
174- 
175- if (ch_idx == CH_RX) {
176- if (!polling_rate)
177- free_irq(dev->client->irq, dev);
178- cancel_delayed_work_sync(&dev->rx.dwork);
179- 
180- while (!list_empty(&dev->rx.list)) {
181- mbo = list_first_mbo(&dev->rx.list);
182- list_del(&mbo->list);
183- 
184- mbo->processed_length = 0;
185- mbo->status = MBO_E_CLOSE;
186- mbo->complete(mbo);
187- }
188- }
189- 
190- return 0;
191-}
192- 
193-static void do_rx_work(struct hdm_i2c *dev)
194-{
195- struct mbo *mbo;
196- unsigned char msg[MAX_BUF_SIZE_CONTROL];
197- int ret;
198- u16 pml, data_size;
199- 
200- /* Read PML (2 bytes) */
201- ret = i2c_master_recv(dev->client, msg, 2);
202- if (ret <= 0) {
203- pr_err("Failed to receive PML\n");
204- return;
205- }
206- 
207- pml = (msg[0] << 8) | msg[1];
208- if (!pml)
209- return;
210- 
211- data_size = pml + 2;
212- 
213- /* Read the whole message, including PML */
214- ret = i2c_master_recv(dev->client, msg, data_size);
215- if (ret <= 0) {
216- pr_err("Failed to receive a Port Message\n");
217- return;
218- }
219- 
220- mbo = list_first_mbo(&dev->rx.list);
221- list_del(&mbo->list);
222- 
223- mbo->processed_length = min(data_size, mbo->buffer_length);
224- memcpy(mbo->virt_address, msg, mbo->processed_length);
225- mbo->status = MBO_SUCCESS;
226- mbo->complete(mbo);
227-}
228- 
229-/**
230- * pending_rx_work - Read pending messages through I2C
231- * @work: definition of this work item
232- *
233- * Invoked by the Interrupt Service Routine, most_irq_handler()
234- */
235-static void pending_rx_work(struct work_struct *work)
236-{
237- struct hdm_i2c *dev = container_of(work, struct hdm_i2c, rx.dwork.work);
238- 
239- if (list_empty(&dev->rx.list))
240- return;
241- 
242- do_rx_work(dev);
243- 
244- if (polling_rate) {
245- schedule_delayed_work(&dev->rx.dwork, dev->rx.delay);
246- } else {
247- dev->rx.int_disabled = false;
248- enable_irq(dev->client->irq);
249- }
250-}
251- 
252-/*
253- * most_irq_handler - Interrupt Service Routine
254- * @irq: irq number
255- * @_dev: private data
256- *
257- * Schedules a delayed work
258- *
259- * By default the interrupt line behavior is Active Low. Once an interrupt is
260- * generated by the device, until driver clears the interrupt (by reading
261- * the PMP message), device keeps the interrupt line in low state. Since i2c
262- * read is done in work queue, the interrupt line must be disabled temporarily
263- * to avoid ISR being called repeatedly. Re-enable the interrupt in workqueue,
264- * after reading the message.
265- *
266- * Note: If we use the interrupt line in Falling edge mode, there is a
267- * possibility to miss interrupts when ISR is getting executed.
268- *
269- */
270-static irqreturn_t most_irq_handler(int irq, void *_dev)
271-{
272- struct hdm_i2c *dev = _dev;
273- 
274- disable_irq_nosync(irq);
275- dev->rx.int_disabled = true;
276- schedule_delayed_work(&dev->rx.dwork, 0);
277- 
278- return IRQ_HANDLED;
279-}
280- 
281-/*
282- * i2c_probe - i2c probe handler
283- * @client: i2c client device structure
284- * @id: i2c client device id
285- *
286- * Return 0 on success, negative on failure.
287- *
288- * Register the i2c client device as a MOST interface
289- */
290-static int i2c_probe(struct i2c_client *client)
291-{
292- struct hdm_i2c *dev;
293- int ret, i;
294- 
295- dev = kzalloc(sizeof(*dev), GFP_KERNEL);
296- if (!dev)
297- return -ENOMEM;
298- 
299- /* ID format: i2c-<bus>-<address> */
300- snprintf(dev->name, sizeof(dev->name), "i2c-%d-%04x",
301- client->adapter->nr, client->addr);
302- 
303- for (i = 0; i < NUM_CHANNELS; i++) {
304- dev->capabilities[i].data_type = MOST_CH_CONTROL;
305- dev->capabilities[i].num_buffers_packet = MAX_BUFFERS_CONTROL;
306- dev->capabilities[i].buffer_size_packet = MAX_BUF_SIZE_CONTROL;
307- }
308- dev->capabilities[CH_RX].direction = MOST_CH_RX;
309- dev->capabilities[CH_RX].name_suffix = "rx";
310- dev->capabilities[CH_TX].direction = MOST_CH_TX;
311- dev->capabilities[CH_TX].name_suffix = "tx";
312- 
313- dev->most_iface.interface = ITYPE_I2C;
314- dev->most_iface.description = dev->name;
315- dev->most_iface.num_channels = NUM_CHANNELS;
316- dev->most_iface.channel_vector = dev->capabilities;
317- dev->most_iface.configure = configure_channel;
318- dev->most_iface.enqueue = enqueue;
319- dev->most_iface.poison_channel = poison_channel;
320- 
321- INIT_LIST_HEAD(&dev->rx.list);
322- 
323- INIT_DELAYED_WORK(&dev->rx.dwork, pending_rx_work);
324- 
325- dev->client = client;
326- i2c_set_clientdata(client, dev);
327- 
328- ret = most_register_interface(&dev->most_iface);
329- if (ret) {
330- pr_err("Failed to register i2c as a MOST interface\n");
331- kfree(dev);
332- return ret;
333- }
334- 
335- return 0;
336-}
337- 
338-/*
339- * i2c_remove - i2c remove handler
340- * @client: i2c client device structure
341- *
342- * Return 0 on success.
343- *
344- * Unregister the i2c client device as a MOST interface
345- */
346-static void i2c_remove(struct i2c_client *client)
347-{
348- struct hdm_i2c *dev = i2c_get_clientdata(client);
349- 
350- most_deregister_interface(&dev->most_iface);
351- kfree(dev);
352-}
353- 
354-static const struct i2c_device_id i2c_id[] = {
355- { "most_i2c", 0 },
356- { }, /* Terminating entry */
357-};
358- 
359-MODULE_DEVICE_TABLE(i2c, i2c_id);
360- 
361-static struct i2c_driver i2c_driver = {
362- .driver = {
363- .name = "hdm_i2c",
364- },
365- .probe = i2c_probe,
366- .remove = i2c_remove,
367- .id_table = i2c_id,
368-};
369- 
370-module_i2c_driver(i2c_driver);
371- 
372-MODULE_AUTHOR("Andrey Shvetsov <andrey.shvetsov@k2l.de>");
373-MODULE_DESCRIPTION("I2C Hardware Dependent Module");
374-MODULE_LICENSE("GPL");
Mdrivers/usb/gadget/function/f_fs.c+7-1
@@ -1941,7 +1941,12 @@ static int ffs_func_eps_enable(struct ffs_function *func)
1941 ep = func->eps;1941 ep = func->eps;
1942 epfile = ffs->epfiles;1942 epfile = ffs->epfiles;
1943 count = ffs->eps_count;1943 count = ffs->eps_count;
1944- while(count--) {1944+ if (!epfile) {
1945+ ret = -ENOMEM;
1946+ goto done;
1947+ }
1948+ 
1949+ while (count--) {
1945 ep->ep->driver_data = ep;1950 ep->ep->driver_data = ep;
1946 1951 
1947 ret = config_ep_by_speed(func->gadget, &func->function, ep->ep);1952 ret = config_ep_by_speed(func->gadget, &func->function, ep->ep);
@@ -1965,6 +1970,7 @@ static int ffs_func_eps_enable(struct ffs_function *func)
1965 }1970 }
1966 1971 
1967 wake_up_interruptible(&ffs->wait);1972 wake_up_interruptible(&ffs->wait);
1973+done:
1968 spin_unlock_irqrestore(&func->ffs->eps_lock, flags);1974 spin_unlock_irqrestore(&func->ffs->eps_lock, flags);
1969 1975 
1970 return ret;1976 return ret;
Mdrivers/video/backlight/led_bl.c+13-0
@@ -209,6 +209,19 @@ static int led_bl_probe(struct platform_device *pdev)
209 return PTR_ERR(priv->bl_dev);209 return PTR_ERR(priv->bl_dev);
210 }210 }
211 211 
212+ for (i = 0; i < priv->nb_leds; i++) {
213+ struct device_link *link;
214+ 
215+ link = device_link_add(&pdev->dev, priv->leds[i]->dev->parent,
216+ DL_FLAG_AUTOREMOVE_CONSUMER);
217+ if (!link) {
218+ dev_err(&pdev->dev, "Failed to add devlink (consumer %s, supplier %s)\n",
219+ dev_name(&pdev->dev), dev_name(priv->leds[i]->dev->parent));
220+ backlight_device_unregister(priv->bl_dev);
221+ return -EINVAL;
222+ }
223+ }
224+ 
212 for (i = 0; i < priv->nb_leds; i++) {225 for (i = 0; i < priv->nb_leds; i++) {
213 mutex_lock(&priv->leds[i]->led_access);226 mutex_lock(&priv->leds[i]->led_access);
214 led_sysfs_disable(priv->leds[i]);227 led_sysfs_disable(priv->leds[i]);
Mdrivers/video/fbdev/core/bitblit.c+17-0
@@ -160,6 +160,11 @@ static void bit_putcs(struct vc_data *vc, struct fb_info *info,
160 image.height = vc->vc_font.height;160 image.height = vc->vc_font.height;
161 image.depth = 1;161 image.depth = 1;
162 162 
163+ if (image.dy >= info->var.yres)
164+ return;
165+ 
166+ image.height = min(image.height, info->var.yres - image.dy);
167+ 
163 if (attribute) {168 if (attribute) {
164 buf = kmalloc(cellsize, GFP_ATOMIC);169 buf = kmalloc(cellsize, GFP_ATOMIC);
165 if (!buf)170 if (!buf)
@@ -173,6 +178,18 @@ static void bit_putcs(struct vc_data *vc, struct fb_info *info,
173 cnt = count;178 cnt = count;
174 179 
175 image.width = vc->vc_font.width * cnt;180 image.width = vc->vc_font.width * cnt;
181+ 
182+ if (image.dx >= info->var.xres)
183+ break;
184+ 
185+ if (image.dx + image.width > info->var.xres) {
186+ image.width = info->var.xres - image.dx;
187+ cnt = image.width / vc->vc_font.width;
188+ if (cnt == 0)
189+ break;
190+ image.width = cnt * vc->vc_font.width;
191+ }
192+ 
176 pitch = DIV_ROUND_UP(image.width, 8) + scan_align;193 pitch = DIV_ROUND_UP(image.width, 8) + scan_align;
177 pitch &= ~scan_align;194 pitch &= ~scan_align;
178 size = pitch * image.height + buf_align;195 size = pitch * image.height + buf_align;
Mdrivers/video/fbdev/core/fbcon.c+19-0
@@ -2816,6 +2816,25 @@ int fbcon_mode_deleted(struct fb_info *info,
2816 return found;2816 return found;
2817}2817}
2818 2818 
2819+static void fbcon_delete_mode(struct fb_videomode *m)
2820+{
2821+ struct fbcon_display *p;
2822+ 
2823+ for (int i = first_fb_vc; i <= last_fb_vc; i++) {
2824+ p = &fb_display[i];
2825+ if (p->mode == m)
2826+ p->mode = NULL;
2827+ }
2828+}
2829+ 
2830+void fbcon_delete_modelist(struct list_head *head)
2831+{
2832+ struct fb_modelist *modelist;
2833+ 
2834+ list_for_each_entry(modelist, head, list)
2835+ fbcon_delete_mode(&modelist->mode);
2836+}
2837+ 
2819#ifdef CONFIG_VT_HW_CONSOLE_BINDING2838#ifdef CONFIG_VT_HW_CONSOLE_BINDING
2820static void fbcon_unbind(void)2839static void fbcon_unbind(void)
2821{2840{
Mdrivers/video/fbdev/core/fbmem.c+1-0
@@ -1027,6 +1027,7 @@ static void do_unregister_framebuffer(struct fb_info *fb_info)
1027 fb_info->pixmap.addr = NULL;1027 fb_info->pixmap.addr = NULL;
1028 }1028 }
1029 1029 
1030+ fbcon_delete_modelist(&fb_info->modelist);
1030 fb_destroy_modelist(&fb_info->modelist);1031 fb_destroy_modelist(&fb_info->modelist);
1031 registered_fb[fb_info->node] = NULL;1032 registered_fb[fb_info->node] = NULL;
1032 num_registered_fb--;1033 num_registered_fb--;
Mfs/btrfs/extent_io.c+9-1
@@ -1696,7 +1696,15 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb,
1696 unlock_page(p);1696 unlock_page(p);
1697 }1697 }
1698 }1698 }
1699- btrfs_submit_bio(bbio, 0);1699+ /*
1700+ * If the fs is already in error status, do not submit any writeback
1701+ * but immediately finish it.
1702+ */
1703+ if (unlikely(BTRFS_FS_ERROR(fs_info))) {
1704+ btrfs_bio_end_io(bbio, errno_to_blk_status(BTRFS_FS_ERROR(fs_info)));
1705+ return;
1706+ }
1707+ btrfs_submit_bbio(bbio, 0);
1700}1708}
1701 1709 
1702/*1710/*
Mfs/btrfs/qgroup.c+25-8
@@ -579,22 +579,30 @@ bool btrfs_check_quota_leak(struct btrfs_fs_info *fs_info)
579 579 
580/*580/*
581 * This is called from close_ctree() or open_ctree() or btrfs_quota_disable(),581 * This is called from close_ctree() or open_ctree() or btrfs_quota_disable(),
582- * first two are in single-threaded paths.And for the third one, we have set582+ * first two are in single-threaded paths.
583- * quota_root to be null with qgroup_lock held before, so it is safe to clean
584- * up the in-memory structures without qgroup_lock held.
585 */583 */
586void btrfs_free_qgroup_config(struct btrfs_fs_info *fs_info)584void btrfs_free_qgroup_config(struct btrfs_fs_info *fs_info)
587{585{
588 struct rb_node *n;586 struct rb_node *n;
589 struct btrfs_qgroup *qgroup;587 struct btrfs_qgroup *qgroup;
590 588 
589+ /*
590+ * btrfs_quota_disable() can be called concurrently with
591+ * btrfs_qgroup_rescan() -> qgroup_rescan_zero_tracking(), so take the
592+ * lock.
593+ */
594+ spin_lock(&fs_info->qgroup_lock);
591 while ((n = rb_first(&fs_info->qgroup_tree))) {595 while ((n = rb_first(&fs_info->qgroup_tree))) {
592 qgroup = rb_entry(n, struct btrfs_qgroup, node);596 qgroup = rb_entry(n, struct btrfs_qgroup, node);
593 rb_erase(n, &fs_info->qgroup_tree);597 rb_erase(n, &fs_info->qgroup_tree);
594- __del_qgroup_rb(fs_info, qgroup);598+ __del_qgroup_rb(qgroup);
599+ spin_unlock(&fs_info->qgroup_lock);
595 btrfs_sysfs_del_one_qgroup(fs_info, qgroup);600 btrfs_sysfs_del_one_qgroup(fs_info, qgroup);
596 kfree(qgroup);601 kfree(qgroup);
602+ spin_lock(&fs_info->qgroup_lock);
597 }603 }
604+ spin_unlock(&fs_info->qgroup_lock);
605+ 
598 /*606 /*
599 * We call btrfs_free_qgroup_config() when unmounting607 * We call btrfs_free_qgroup_config() when unmounting
600 * filesystem and disabling quota, so we set qgroup_ulist608 * filesystem and disabling quota, so we set qgroup_ulist
@@ -3616,12 +3624,21 @@ btrfs_qgroup_rescan(struct btrfs_fs_info *fs_info)
3616 qgroup_rescan_zero_tracking(fs_info);3624 qgroup_rescan_zero_tracking(fs_info);
3617 3625 
3618 mutex_lock(&fs_info->qgroup_rescan_lock);3626 mutex_lock(&fs_info->qgroup_rescan_lock);
3619- fs_info->qgroup_rescan_running = true;3627+ /*
3620- btrfs_queue_work(fs_info->qgroup_rescan_workers,3628+ * The rescan worker is only for full accounting qgroups, check if it's
3621- &fs_info->qgroup_rescan_work);3629+ * enabled as it is pointless to queue it otherwise. A concurrent quota
3630+ * disable may also have just cleared BTRFS_FS_QUOTA_ENABLED.
3631+ */
3632+ if (btrfs_qgroup_full_accounting(fs_info)) {
3633+ fs_info->qgroup_rescan_running = true;
3634+ btrfs_queue_work(fs_info->qgroup_rescan_workers,
3635+ &fs_info->qgroup_rescan_work);
3636+ } else {
3637+ ret = -ENOTCONN;
3638+ }
3622 mutex_unlock(&fs_info->qgroup_rescan_lock);3639 mutex_unlock(&fs_info->qgroup_rescan_lock);
3623 3640 
3624- return 0;3641+ return ret;
3625}3642}
3626 3643 
3627int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info,3644int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info,
Mfs/btrfs/tree-log.c+38-0
@@ -5661,6 +5661,33 @@ static int conflicting_inode_is_dir(struct btrfs_root *root, u64 ino,
5661 return ret;5661 return ret;
5662}5662}
5663 5663 
5664+static bool can_log_conflicting_inode(const struct btrfs_trans_handle *trans,
5665+ const struct btrfs_inode *inode)
5666+{
5667+ if (!S_ISDIR(inode->vfs_inode.i_mode))
5668+ return true;
5669+ 
5670+ if (inode->last_unlink_trans < trans->transid)
5671+ return true;
5672+ 
5673+ /*
5674+ * If this is a directory and its unlink_trans is not from a past
5675+ * transaction then we must fallback to a transaction commit in order
5676+ * to avoid getting a directory with 2 hard links after log replay.
5677+ *
5678+ * This happens if a directory A is renamed, moved from one parent
5679+ * directory to another one, a new file is created in the old parent
5680+ * directory with the old name of our directory A, the new file is
5681+ * fsynced, then we moved the new file to some other parent directory
5682+ * and fsync again the new file. This results in a log tree where we
5683+ * logged that directory A existed, with the INODE_REF item for the
5684+ * new location but without having logged its old parent inode, so
5685+ * that on log replay we add a new link for the new location but the
5686+ * old link remains, resulting in a link count of 2.
5687+ */
5688+ return false;
5689+}
5690+ 
5664static int add_conflicting_inode(struct btrfs_trans_handle *trans,5691static int add_conflicting_inode(struct btrfs_trans_handle *trans,
5665 struct btrfs_root *root,5692 struct btrfs_root *root,
5666 struct btrfs_path *path,5693 struct btrfs_path *path,
@@ -5764,6 +5791,11 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans,
5764 return 0;5791 return 0;
5765 }5792 }
5766 5793 
5794+ if (!can_log_conflicting_inode(trans, inode)) {
5795+ btrfs_add_delayed_iput(inode);
5796+ return BTRFS_LOG_FORCE_COMMIT;
5797+ }
5798+ 
5767 btrfs_add_delayed_iput(inode);5799 btrfs_add_delayed_iput(inode);
5768 5800 
5769 ino_elem = kmalloc(sizeof(*ino_elem), GFP_NOFS);5801 ino_elem = kmalloc(sizeof(*ino_elem), GFP_NOFS);
@@ -5828,6 +5860,12 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans,
5828 break;5860 break;
5829 }5861 }
5830 5862 
5863+ if (!can_log_conflicting_inode(trans, inode)) {
5864+ btrfs_add_delayed_iput(inode);
5865+ ret = BTRFS_LOG_FORCE_COMMIT;
5866+ break;
5867+ }
5868+ 
5831 /*5869 /*
5832 * Always log the directory, we cannot make this5870 * Always log the directory, we cannot make this
5833 * conditional on need_log_inode() because the directory5871 * conditional on need_log_inode() because the directory
Mfs/ceph/mds_client.c+906-365
Mfs/ceph/mdsmap.c+13-1
@@ -353,10 +353,22 @@ struct ceph_mdsmap *ceph_mdsmap_decode(struct ceph_mds_client *mdsc, void **p,
353 __decode_and_drop_type(p, end, u8, bad_ext);353 __decode_and_drop_type(p, end, u8, bad_ext);
354 }354 }
355 if (mdsmap_ev >= 8) {355 if (mdsmap_ev >= 8) {
356+ u32 fsname_len;
356 /* enabled */357 /* enabled */
357 ceph_decode_8_safe(p, end, m->m_enabled, bad_ext);358 ceph_decode_8_safe(p, end, m->m_enabled, bad_ext);
358 /* fs_name */359 /* fs_name */
359- ceph_decode_skip_string(p, end, bad_ext);360+ ceph_decode_32_safe(p, end, fsname_len, bad_ext);
361+ 
362+ /* validate fsname against mds_namespace */
363+ if (!namespace_equals(mdsc->fsc->mount_options, *p,
364+ fsname_len)) {
365+ pr_warn_client(cl, "fsname %*pE doesn't match mds_namespace %s\n",
366+ (int)fsname_len, (char *)*p,
367+ mdsc->fsc->mount_options->mds_namespace);
368+ goto bad;
369+ }
370+ /* skip fsname after validation */
371+ ceph_decode_skip_n(p, end, fsname_len, bad);
360 }372 }
361 /* damaged */373 /* damaged */
362 if (mdsmap_ev >= 9) {374 if (mdsmap_ev >= 9) {
Mfs/ceph/super.c+0-14
@@ -242,20 +242,6 @@ static void canonicalize_path(char *path)
242 path[j] = '\0';242 path[j] = '\0';
243}243}
244 244 
245-/*
246- * Check if the mds namespace in ceph_mount_options matches
247- * the passed in namespace string. First time match (when
248- * ->mds_namespace is NULL) is treated specially, since
249- * ->mds_namespace needs to be initialized by the caller.
250- */
251-static int namespace_equals(struct ceph_mount_options *fsopt,
252- const char *namespace, size_t len)
253-{
254- return !(fsopt->mds_namespace &&
255- (strlen(fsopt->mds_namespace) != len ||
256- strncmp(fsopt->mds_namespace, namespace, len)));
257-}
258- 
259static int ceph_parse_old_source(const char *dev_name, const char *dev_name_end,245static int ceph_parse_old_source(const char *dev_name, const char *dev_name_end,
260 struct fs_context *fc)246 struct fs_context *fc)
261{247{
Mfs/ceph/super.h+14-0
@@ -104,6 +104,20 @@ struct ceph_mount_options {
104 struct fscrypt_dummy_policy dummy_enc_policy;104 struct fscrypt_dummy_policy dummy_enc_policy;
105};105};
106 106 
107+/*
108+ * Check if the mds namespace in ceph_mount_options matches
109+ * the passed in namespace string. First time match (when
110+ * ->mds_namespace is NULL) is treated specially, since
111+ * ->mds_namespace needs to be initialized by the caller.
112+ */
113+static inline int namespace_equals(struct ceph_mount_options *fsopt,
114+ const char *namespace, size_t len)
115+{
116+ return !(fsopt->mds_namespace &&
117+ (strlen(fsopt->mds_namespace) != len ||
118+ strncmp(fsopt->mds_namespace, namespace, len)));
119+}
120+ 
107/* mount state */121/* mount state */
108enum {122enum {
109 CEPH_MOUNT_MOUNTING,123 CEPH_MOUNT_MOUNTING,
Mfs/f2fs/compress.c+344-1214
@@ -7,55 +7,20 @@
7 7 
8#include <linux/fs.h>8#include <linux/fs.h>
9#include <linux/f2fs_fs.h>9#include <linux/f2fs_fs.h>
10-#include <linux/moduleparam.h>
11#include <linux/writeback.h>10#include <linux/writeback.h>
12#include <linux/backing-dev.h>11#include <linux/backing-dev.h>
13#include <linux/lzo.h>12#include <linux/lzo.h>
14#include <linux/lz4.h>13#include <linux/lz4.h>
15-#include <linux/zstd.h>
16-#include <linux/pagevec.h>
17 14 
18#include "f2fs.h"15#include "f2fs.h"
19#include "node.h"16#include "node.h"
20-#include "segment.h"
21#include <trace/events/f2fs.h>17#include <trace/events/f2fs.h>
22 18 
23-static struct kmem_cache *cic_entry_slab;
24-static struct kmem_cache *dic_entry_slab;
25- 
26-static void *page_array_alloc(struct inode *inode, int nr)
27-{
28- struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
29- unsigned int size = sizeof(struct page *) * nr;
30- 
31- if (likely(size <= sbi->page_array_slab_size))
32- return f2fs_kmem_cache_alloc(sbi->page_array_slab,
33- GFP_F2FS_ZERO, false, F2FS_I_SB(inode));
34- return f2fs_kzalloc(sbi, size, GFP_NOFS);
35-}
36- 
37-static void page_array_free(struct inode *inode, void *pages, int nr)
38-{
39- struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
40- unsigned int size = sizeof(struct page *) * nr;
41- 
42- if (!pages)
43- return;
44- 
45- if (likely(size <= sbi->page_array_slab_size))
46- kmem_cache_free(sbi->page_array_slab, pages);
47- else
48- kfree(pages);
49-}
50- 
51struct f2fs_compress_ops {19struct f2fs_compress_ops {
52 int (*init_compress_ctx)(struct compress_ctx *cc);20 int (*init_compress_ctx)(struct compress_ctx *cc);
53 void (*destroy_compress_ctx)(struct compress_ctx *cc);21 void (*destroy_compress_ctx)(struct compress_ctx *cc);
54 int (*compress_pages)(struct compress_ctx *cc);22 int (*compress_pages)(struct compress_ctx *cc);
55- int (*init_decompress_ctx)(struct decompress_io_ctx *dic);
56- void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic);
57 int (*decompress_pages)(struct decompress_io_ctx *dic);23 int (*decompress_pages)(struct decompress_io_ctx *dic);
58- bool (*is_level_valid)(int level);
59};24};
60 25 
61static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index)26static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index)
@@ -79,22 +44,33 @@ bool f2fs_is_compressed_page(struct page *page)
79 return false;44 return false;
80 if (!page_private(page))45 if (!page_private(page))
81 return false;46 return false;
82- if (page_private_nonpointer(page))47+ if (IS_ATOMIC_WRITTEN_PAGE(page) || IS_DUMMY_WRITTEN_PAGE(page))
83 return false;48 return false;
84- 
85 f2fs_bug_on(F2FS_M_SB(page->mapping),49 f2fs_bug_on(F2FS_M_SB(page->mapping),
86 *((u32 *)page_private(page)) != F2FS_COMPRESSED_PAGE_MAGIC);50 *((u32 *)page_private(page)) != F2FS_COMPRESSED_PAGE_MAGIC);
87 return true;51 return true;
88}52}
89 53 
90static void f2fs_set_compressed_page(struct page *page,54static void f2fs_set_compressed_page(struct page *page,
91- struct inode *inode, pgoff_t index, void *data)55+ struct inode *inode, pgoff_t index, void *data, refcount_t *r)
92{56{
93- attach_page_private(page, (void *)data);57+ SetPagePrivate(page);
58+ set_page_private(page, (unsigned long)data);
94 59 
95 /* i_crypto_info and iv index */60 /* i_crypto_info and iv index */
96 page->index = index;61 page->index = index;
97 page->mapping = inode->i_mapping;62 page->mapping = inode->i_mapping;
63+ if (r)
64+ refcount_inc(r);
65+}
66+ 
67+static void f2fs_put_compressed_page(struct page *page)
68+{
69+ set_page_private(page, (unsigned long)NULL);
70+ ClearPagePrivate(page);
71+ page->mapping = NULL;
72+ unlock_page(page);
73+ put_page(page);
98}74}
99 75 
100static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock)76static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock)
@@ -121,6 +97,20 @@ static void f2fs_unlock_rpages(struct compress_ctx *cc, int len)
121 f2fs_drop_rpages(cc, len, true);97 f2fs_drop_rpages(cc, len, true);
122}98}
123 99 
100+static void f2fs_put_rpages_mapping(struct compress_ctx *cc,
101+ struct address_space *mapping,
102+ pgoff_t start, int len)
103+{
104+ int i;
105+ 
106+ for (i = 0; i < len; i++) {
107+ struct page *page = find_get_page(mapping, start + i);
108+ 
109+ put_page(page);
110+ put_page(page);
111+ }
112+}
113+ 
124static void f2fs_put_rpages_wbc(struct compress_ctx *cc,114static void f2fs_put_rpages_wbc(struct compress_ctx *cc,
125 struct writeback_control *wbc, bool redirty, int unlock)115 struct writeback_control *wbc, bool redirty, int unlock)
126{116{
@@ -142,22 +132,23 @@ struct page *f2fs_compress_control_page(struct page *page)
142 132 
143int f2fs_init_compress_ctx(struct compress_ctx *cc)133int f2fs_init_compress_ctx(struct compress_ctx *cc)
144{134{
145- if (cc->rpages)135+ struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
136+ 
137+ if (cc->nr_rpages)
146 return 0;138 return 0;
147 139 
148- cc->rpages = page_array_alloc(cc->inode, cc->cluster_size);140+ cc->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
141+ cc->log_cluster_size, GFP_NOFS);
149 return cc->rpages ? 0 : -ENOMEM;142 return cc->rpages ? 0 : -ENOMEM;
150}143}
151 144 
152-void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse)145+void f2fs_destroy_compress_ctx(struct compress_ctx *cc)
153{146{
154- page_array_free(cc->inode, cc->rpages, cc->cluster_size);147+ kfree(cc->rpages);
155 cc->rpages = NULL;148 cc->rpages = NULL;
156 cc->nr_rpages = 0;149 cc->nr_rpages = 0;
157 cc->nr_cpages = 0;150 cc->nr_cpages = 0;
158- cc->valid_nr_cpages = 0;151+ cc->cluster_idx = NULL_CLUSTER;
159- if (!reuse)
160- cc->cluster_idx = NULL_CLUSTER;
161}152}
162 153 
163void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page)154void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page)
@@ -176,7 +167,8 @@ void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page)
176#ifdef CONFIG_F2FS_FS_LZO167#ifdef CONFIG_F2FS_FS_LZO
177static int lzo_init_compress_ctx(struct compress_ctx *cc)168static int lzo_init_compress_ctx(struct compress_ctx *cc)
178{169{
179- cc->private = f2fs_vmalloc(LZO1X_MEM_COMPRESS);170+ cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
171+ LZO1X_MEM_COMPRESS, GFP_NOFS);
180 if (!cc->private)172 if (!cc->private)
181 return -ENOMEM;173 return -ENOMEM;
182 174 
@@ -186,7 +178,7 @@ static int lzo_init_compress_ctx(struct compress_ctx *cc)
186 178 
187static void lzo_destroy_compress_ctx(struct compress_ctx *cc)179static void lzo_destroy_compress_ctx(struct compress_ctx *cc)
188{180{
189- vfree(cc->private);181+ kvfree(cc->private);
190 cc->private = NULL;182 cc->private = NULL;
191}183}
192 184 
@@ -197,8 +189,8 @@ static int lzo_compress_pages(struct compress_ctx *cc)
197 ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,189 ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
198 &cc->clen, cc->private);190 &cc->clen, cc->private);
199 if (ret != LZO_E_OK) {191 if (ret != LZO_E_OK) {
200- f2fs_err_ratelimited(F2FS_I_SB(cc->inode),192+ printk_ratelimited("%sF2FS-fs (%s): lzo compress failed, ret:%d\n",
201- "lzo compress failed, ret:%d", ret);193+ KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, ret);
202 return -EIO;194 return -EIO;
203 }195 }
204 return 0;196 return 0;
@@ -211,15 +203,17 @@ static int lzo_decompress_pages(struct decompress_io_ctx *dic)
211 ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen,203 ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen,
212 dic->rbuf, &dic->rlen);204 dic->rbuf, &dic->rlen);
213 if (ret != LZO_E_OK) {205 if (ret != LZO_E_OK) {
214- f2fs_err_ratelimited(F2FS_I_SB(dic->inode),206+ printk_ratelimited("%sF2FS-fs (%s): lzo decompress failed, ret:%d\n",
215- "lzo decompress failed, ret:%d", ret);207+ KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);
216 return -EIO;208 return -EIO;
217 }209 }
218 210 
219 if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) {211 if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) {
220- f2fs_err_ratelimited(F2FS_I_SB(dic->inode),212+ printk_ratelimited("%sF2FS-fs (%s): lzo invalid rlen:%zu, "
221- "lzo invalid rlen:%zu, expected:%lu",213+ "expected:%lu\n", KERN_ERR,
222- dic->rlen, PAGE_SIZE << dic->log_cluster_size);214+ F2FS_I_SB(dic->inode)->sb->s_id,
215+ dic->rlen,
216+ PAGE_SIZE << dic->log_cluster_size);
223 return -EIO;217 return -EIO;
224 }218 }
225 return 0;219 return 0;
@@ -236,50 +230,32 @@ static const struct f2fs_compress_ops f2fs_lzo_ops = {
236#ifdef CONFIG_F2FS_FS_LZ4230#ifdef CONFIG_F2FS_FS_LZ4
237static int lz4_init_compress_ctx(struct compress_ctx *cc)231static int lz4_init_compress_ctx(struct compress_ctx *cc)
238{232{
239- unsigned int size = LZ4_MEM_COMPRESS;233+ cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
240- 234+ LZ4_MEM_COMPRESS, GFP_NOFS);
241-#ifdef CONFIG_F2FS_FS_LZ4HC
242- if (F2FS_I(cc->inode)->i_compress_level)
243- size = LZ4HC_MEM_COMPRESS;
244-#endif
245- 
246- cc->private = f2fs_vmalloc(size);
247 if (!cc->private)235 if (!cc->private)
248 return -ENOMEM;236 return -ENOMEM;
249 237 
250- /*238+ cc->clen = LZ4_compressBound(PAGE_SIZE << cc->log_cluster_size);
251- * we do not change cc->clen to LZ4_compressBound(inputsize) to
252- * adapt worst compress case, because lz4 compressor can handle
253- * output budget properly.
254- */
255- cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE;
256 return 0;239 return 0;
257}240}
258 241 
259static void lz4_destroy_compress_ctx(struct compress_ctx *cc)242static void lz4_destroy_compress_ctx(struct compress_ctx *cc)
260{243{
261- vfree(cc->private);244+ kvfree(cc->private);
262 cc->private = NULL;245 cc->private = NULL;
263}246}
264 247 
265static int lz4_compress_pages(struct compress_ctx *cc)248static int lz4_compress_pages(struct compress_ctx *cc)
266{249{
267- int len = -EINVAL;250+ int len;
268- unsigned char level = F2FS_I(cc->inode)->i_compress_level;
269 251 
270- if (!level)252+ len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen,
271- len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen,
272 cc->clen, cc->private);253 cc->clen, cc->private);
273-#ifdef CONFIG_F2FS_FS_LZ4HC254+ if (!len) {
274- else255+ printk_ratelimited("%sF2FS-fs (%s): lz4 compress failed\n",
275- len = LZ4_compress_HC(cc->rbuf, cc->cbuf->cdata, cc->rlen,256+ KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id);
276- cc->clen, level, cc->private);257+ return -EIO;
277-#endif258+ }
278- if (len < 0)
279- return len;
280- if (!len)
281- return -EAGAIN;
282- 
283 cc->clen = len;259 cc->clen = len;
284 return 0;260 return 0;
285}261}
@@ -291,234 +267,30 @@ static int lz4_decompress_pages(struct decompress_io_ctx *dic)
291 ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf,267 ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf,
292 dic->clen, dic->rlen);268 dic->clen, dic->rlen);
293 if (ret < 0) {269 if (ret < 0) {
294- f2fs_err_ratelimited(F2FS_I_SB(dic->inode),270+ printk_ratelimited("%sF2FS-fs (%s): lz4 decompress failed, ret:%d\n",
295- "lz4 decompress failed, ret:%d", ret);271+ KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);
296 return -EIO;272 return -EIO;
297 }273 }
298 274 
299 if (ret != PAGE_SIZE << dic->log_cluster_size) {275 if (ret != PAGE_SIZE << dic->log_cluster_size) {
300- f2fs_err_ratelimited(F2FS_I_SB(dic->inode),276+ printk_ratelimited("%sF2FS-fs (%s): lz4 invalid rlen:%zu, "
301- "lz4 invalid ret:%d, expected:%lu",277+ "expected:%lu\n", KERN_ERR,
302- ret, PAGE_SIZE << dic->log_cluster_size);278+ F2FS_I_SB(dic->inode)->sb->s_id,
279+ dic->rlen,
280+ PAGE_SIZE << dic->log_cluster_size);
303 return -EIO;281 return -EIO;
304 }282 }
305 return 0;283 return 0;
306}284}
307 285 
308-static bool lz4_is_level_valid(int lvl)
309-{
310-#ifdef CONFIG_F2FS_FS_LZ4HC
311- return !lvl || (lvl >= LZ4HC_MIN_CLEVEL && lvl <= LZ4HC_MAX_CLEVEL);
312-#else
313- return lvl == 0;
314-#endif
315-}
316- 
317static const struct f2fs_compress_ops f2fs_lz4_ops = {286static const struct f2fs_compress_ops f2fs_lz4_ops = {
318 .init_compress_ctx = lz4_init_compress_ctx,287 .init_compress_ctx = lz4_init_compress_ctx,
319 .destroy_compress_ctx = lz4_destroy_compress_ctx,288 .destroy_compress_ctx = lz4_destroy_compress_ctx,
320 .compress_pages = lz4_compress_pages,289 .compress_pages = lz4_compress_pages,
321 .decompress_pages = lz4_decompress_pages,290 .decompress_pages = lz4_decompress_pages,
322- .is_level_valid = lz4_is_level_valid,
323};291};
324#endif292#endif
325 293 
326-#ifdef CONFIG_F2FS_FS_ZSTD
327-static int zstd_init_compress_ctx(struct compress_ctx *cc)
328-{
329- zstd_parameters params;
330- zstd_cstream *stream;
331- void *workspace;
332- unsigned int workspace_size;
333- unsigned char level = F2FS_I(cc->inode)->i_compress_level;
334- 
335- /* Need to remain this for backward compatibility */
336- if (!level)
337- level = F2FS_ZSTD_DEFAULT_CLEVEL;
338- 
339- params = zstd_get_params(level, cc->rlen);
340- workspace_size = zstd_cstream_workspace_bound(&params.cParams);
341- 
342- workspace = f2fs_vmalloc(workspace_size);
343- if (!workspace)
344- return -ENOMEM;
345- 
346- stream = zstd_init_cstream(&params, 0, workspace, workspace_size);
347- if (!stream) {
348- f2fs_err_ratelimited(F2FS_I_SB(cc->inode),
349- "%s zstd_init_cstream failed", __func__);
350- vfree(workspace);
351- return -EIO;
352- }
353- 
354- cc->private = workspace;
355- cc->private2 = stream;
356- 
357- cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE;
358- return 0;
359-}
360- 
361-static void zstd_destroy_compress_ctx(struct compress_ctx *cc)
362-{
363- vfree(cc->private);
364- cc->private = NULL;
365- cc->private2 = NULL;
366-}
367- 
368-static int zstd_compress_pages(struct compress_ctx *cc)
369-{
370- zstd_cstream *stream = cc->private2;
371- zstd_in_buffer inbuf;
372- zstd_out_buffer outbuf;
373- int src_size = cc->rlen;
374- int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE;
375- int ret;
376- 
377- inbuf.pos = 0;
378- inbuf.src = cc->rbuf;
379- inbuf.size = src_size;
380- 
381- outbuf.pos = 0;
382- outbuf.dst = cc->cbuf->cdata;
383- outbuf.size = dst_size;
384- 
385- ret = zstd_compress_stream(stream, &outbuf, &inbuf);
386- if (zstd_is_error(ret)) {
387- f2fs_err_ratelimited(F2FS_I_SB(cc->inode),
388- "%s zstd_compress_stream failed, ret: %d",
389- __func__, zstd_get_error_code(ret));
390- return -EIO;
391- }
392- 
393- ret = zstd_end_stream(stream, &outbuf);
394- if (zstd_is_error(ret)) {
395- f2fs_err_ratelimited(F2FS_I_SB(cc->inode),
396- "%s zstd_end_stream returned %d",
397- __func__, zstd_get_error_code(ret));
398- return -EIO;
399- }
400- 
401- /*
402- * there is compressed data remained in intermediate buffer due to
403- * no more space in cbuf.cdata
404- */
405- if (ret)
406- return -EAGAIN;
407- 
408- cc->clen = outbuf.pos;
409- return 0;
410-}
411- 
412-static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic)
413-{
414- zstd_dstream *stream;
415- void *workspace;
416- unsigned int workspace_size;
417- unsigned int max_window_size =
418- MAX_COMPRESS_WINDOW_SIZE(dic->log_cluster_size);
419- 
420- workspace_size = zstd_dstream_workspace_bound(max_window_size);
421- 
422- workspace = f2fs_vmalloc(workspace_size);
423- if (!workspace)
424- return -ENOMEM;
425- 
426- stream = zstd_init_dstream(max_window_size, workspace, workspace_size);
427- if (!stream) {
428- f2fs_err_ratelimited(F2FS_I_SB(dic->inode),
429- "%s zstd_init_dstream failed", __func__);
430- vfree(workspace);
431- return -EIO;
432- }
433- 
434- dic->private = workspace;
435- dic->private2 = stream;
436- 
437- return 0;
438-}
439- 
440-static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic)
441-{
442- vfree(dic->private);
443- dic->private = NULL;
444- dic->private2 = NULL;
445-}
446- 
447-static int zstd_decompress_pages(struct decompress_io_ctx *dic)
448-{
449- zstd_dstream *stream = dic->private2;
450- zstd_in_buffer inbuf;
451- zstd_out_buffer outbuf;
452- int ret;
453- 
454- inbuf.pos = 0;
455- inbuf.src = dic->cbuf->cdata;
456- inbuf.size = dic->clen;
457- 
458- outbuf.pos = 0;
459- outbuf.dst = dic->rbuf;
460- outbuf.size = dic->rlen;
461- 
462- ret = zstd_decompress_stream(stream, &outbuf, &inbuf);
463- if (zstd_is_error(ret)) {
464- f2fs_err_ratelimited(F2FS_I_SB(dic->inode),
465- "%s zstd_decompress_stream failed, ret: %d",
466- __func__, zstd_get_error_code(ret));
467- return -EIO;
468- }
469- 
470- if (dic->rlen != outbuf.pos) {
471- f2fs_err_ratelimited(F2FS_I_SB(dic->inode),
472- "%s ZSTD invalid rlen:%zu, expected:%lu",
473- __func__, dic->rlen,
474- PAGE_SIZE << dic->log_cluster_size);
475- return -EIO;
476- }
477- 
478- return 0;
479-}
480- 
481-static bool zstd_is_level_valid(int lvl)
482-{
483- return lvl >= zstd_min_clevel() && lvl <= zstd_max_clevel();
484-}
485- 
486-static const struct f2fs_compress_ops f2fs_zstd_ops = {
487- .init_compress_ctx = zstd_init_compress_ctx,
488- .destroy_compress_ctx = zstd_destroy_compress_ctx,
489- .compress_pages = zstd_compress_pages,
490- .init_decompress_ctx = zstd_init_decompress_ctx,
491- .destroy_decompress_ctx = zstd_destroy_decompress_ctx,
492- .decompress_pages = zstd_decompress_pages,
493- .is_level_valid = zstd_is_level_valid,
494-};
495-#endif
496- 
497-#ifdef CONFIG_F2FS_FS_LZO
498-#ifdef CONFIG_F2FS_FS_LZORLE
499-static int lzorle_compress_pages(struct compress_ctx *cc)
500-{
501- int ret;
502- 
503- ret = lzorle1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
504- &cc->clen, cc->private);
505- if (ret != LZO_E_OK) {
506- f2fs_err_ratelimited(F2FS_I_SB(cc->inode),
507- "lzo-rle compress failed, ret:%d", ret);
508- return -EIO;
509- }
510- return 0;
511-}
512- 
513-static const struct f2fs_compress_ops f2fs_lzorle_ops = {
514- .init_compress_ctx = lzo_init_compress_ctx,
515- .destroy_compress_ctx = lzo_destroy_compress_ctx,
516- .compress_pages = lzorle_compress_pages,
517- .decompress_pages = lzo_decompress_pages,
518-};
519-#endif
520-#endif
521- 
522static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = {294static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = {
523#ifdef CONFIG_F2FS_FS_LZO295#ifdef CONFIG_F2FS_FS_LZO
524 &f2fs_lzo_ops,296 &f2fs_lzo_ops,
@@ -530,16 +302,6 @@ static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = {
530#else302#else
531 NULL,303 NULL,
532#endif304#endif
533-#ifdef CONFIG_F2FS_FS_ZSTD
534- &f2fs_zstd_ops,
535-#else
536- NULL,
537-#endif
538-#if defined(CONFIG_F2FS_FS_LZO) && defined(CONFIG_F2FS_FS_LZORLE)
539- &f2fs_lzorle_ops,
540-#else
541- NULL,
542-#endif
543};305};
544 306 
545bool f2fs_is_compress_backend_ready(struct inode *inode)307bool f2fs_is_compress_backend_ready(struct inode *inode)
@@ -549,107 +311,58 @@ bool f2fs_is_compress_backend_ready(struct inode *inode)
549 return f2fs_cops[F2FS_I(inode)->i_compress_algorithm];311 return f2fs_cops[F2FS_I(inode)->i_compress_algorithm];
550}312}
551 313 
552-bool f2fs_is_compress_level_valid(int alg, int lvl)314+static struct page *f2fs_grab_page(void)
553-{
554- const struct f2fs_compress_ops *cops = f2fs_cops[alg];
555- 
556- if (cops->is_level_valid)
557- return cops->is_level_valid(lvl);
558- 
559- return lvl == 0;
560-}
561- 
562-static mempool_t *compress_page_pool;
563-static int num_compress_pages = 512;
564-module_param(num_compress_pages, uint, 0444);
565-MODULE_PARM_DESC(num_compress_pages,
566- "Number of intermediate compress pages to preallocate");
567- 
568-int __init f2fs_init_compress_mempool(void)
569-{
570- compress_page_pool = mempool_create_page_pool(num_compress_pages, 0);
571- return compress_page_pool ? 0 : -ENOMEM;
572-}
573- 
574-void f2fs_destroy_compress_mempool(void)
575-{
576- mempool_destroy(compress_page_pool);
577-}
578- 
579-static struct page *f2fs_compress_alloc_page(void)
580{315{
581 struct page *page;316 struct page *page;
582 317 
583- page = mempool_alloc(compress_page_pool, GFP_NOFS);318+ page = alloc_page(GFP_NOFS);
584- lock_page(page);
585- 
586- return page;
587-}
588- 
589-static void f2fs_compress_free_page(struct page *page)
590-{
591 if (!page)319 if (!page)
592- return;320+ return NULL;
593- detach_page_private(page);321+ lock_page(page);
594- page->mapping = NULL;322+ return page;
595- unlock_page(page);
596- mempool_free(page, compress_page_pool);
597-}
598- 
599-#define MAX_VMAP_RETRIES 3
600- 
601-static void *f2fs_vmap(struct page **pages, unsigned int count)
602-{
603- int i;
604- void *buf = NULL;
605- 
606- for (i = 0; i < MAX_VMAP_RETRIES; i++) {
607- buf = vm_map_ram(pages, count, -1);
608- if (buf)
609- break;
610- vm_unmap_aliases();
611- }
612- return buf;
613}323}
614 324 
615static int f2fs_compress_pages(struct compress_ctx *cc)325static int f2fs_compress_pages(struct compress_ctx *cc)
616{326{
327+ struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
617 struct f2fs_inode_info *fi = F2FS_I(cc->inode);328 struct f2fs_inode_info *fi = F2FS_I(cc->inode);
618 const struct f2fs_compress_ops *cops =329 const struct f2fs_compress_ops *cops =
619 f2fs_cops[fi->i_compress_algorithm];330 f2fs_cops[fi->i_compress_algorithm];
620- unsigned int max_len, new_nr_cpages;331+ unsigned int max_len, nr_cpages;
621- u32 chksum = 0;
622 int i, ret;332 int i, ret;
623 333 
624 trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx,334 trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx,
625 cc->cluster_size, fi->i_compress_algorithm);335 cc->cluster_size, fi->i_compress_algorithm);
626 336 
627- if (cops->init_compress_ctx) {337+ ret = cops->init_compress_ctx(cc);
628- ret = cops->init_compress_ctx(cc);338+ if (ret)
629- if (ret)339+ goto out;
630- goto out;
631- }
632 340 
633 max_len = COMPRESS_HEADER_SIZE + cc->clen;341 max_len = COMPRESS_HEADER_SIZE + cc->clen;
634 cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE);342 cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE);
635- cc->valid_nr_cpages = cc->nr_cpages;
636 343 
637- cc->cpages = page_array_alloc(cc->inode, cc->nr_cpages);344+ cc->cpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
345+ cc->nr_cpages, GFP_NOFS);
638 if (!cc->cpages) {346 if (!cc->cpages) {
639 ret = -ENOMEM;347 ret = -ENOMEM;
640 goto destroy_compress_ctx;348 goto destroy_compress_ctx;
641 }349 }
642 350 
643- for (i = 0; i < cc->nr_cpages; i++)351+ for (i = 0; i < cc->nr_cpages; i++) {
644- cc->cpages[i] = f2fs_compress_alloc_page();352+ cc->cpages[i] = f2fs_grab_page();
353+ if (!cc->cpages[i]) {
354+ ret = -ENOMEM;
355+ goto out_free_cpages;
356+ }
357+ }
645 358 
646- cc->rbuf = f2fs_vmap(cc->rpages, cc->cluster_size);359+ cc->rbuf = vmap(cc->rpages, cc->cluster_size, VM_MAP, PAGE_KERNEL_RO);
647 if (!cc->rbuf) {360 if (!cc->rbuf) {
648 ret = -ENOMEM;361 ret = -ENOMEM;
649 goto out_free_cpages;362 goto out_free_cpages;
650 }363 }
651 364 
652- cc->cbuf = f2fs_vmap(cc->cpages, cc->nr_cpages);365+ cc->cbuf = vmap(cc->cpages, cc->nr_cpages, VM_MAP, PAGE_KERNEL);
653 if (!cc->cbuf) {366 if (!cc->cbuf) {
654 ret = -ENOMEM;367 ret = -ENOMEM;
655 goto out_vunmap_rbuf;368 goto out_vunmap_rbuf;
@@ -668,84 +381,85 @@ static int f2fs_compress_pages(struct compress_ctx *cc)
668 381 
669 cc->cbuf->clen = cpu_to_le32(cc->clen);382 cc->cbuf->clen = cpu_to_le32(cc->clen);
670 383 
671- if (fi->i_compress_flag & BIT(COMPRESS_CHKSUM))
672- chksum = f2fs_crc32(F2FS_I_SB(cc->inode),
673- cc->cbuf->cdata, cc->clen);
674- cc->cbuf->chksum = cpu_to_le32(chksum);
675- 
676 for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++)384 for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++)
677 cc->cbuf->reserved[i] = cpu_to_le32(0);385 cc->cbuf->reserved[i] = cpu_to_le32(0);
678 386 
679- new_nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE);387+ nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE);
680 388 
681 /* zero out any unused part of the last page */389 /* zero out any unused part of the last page */
682 memset(&cc->cbuf->cdata[cc->clen], 0,390 memset(&cc->cbuf->cdata[cc->clen], 0,
683- (new_nr_cpages * PAGE_SIZE) -391+ (nr_cpages * PAGE_SIZE) - (cc->clen + COMPRESS_HEADER_SIZE));
684- (cc->clen + COMPRESS_HEADER_SIZE));
685 392 
686- vm_unmap_ram(cc->cbuf, cc->nr_cpages);393+ vunmap(cc->cbuf);
687- vm_unmap_ram(cc->rbuf, cc->cluster_size);394+ vunmap(cc->rbuf);
688 395 
689- for (i = new_nr_cpages; i < cc->nr_cpages; i++) {396+ for (i = nr_cpages; i < cc->nr_cpages; i++) {
690- f2fs_compress_free_page(cc->cpages[i]);397+ f2fs_put_compressed_page(cc->cpages[i]);
691 cc->cpages[i] = NULL;398 cc->cpages[i] = NULL;
692 }399 }
693 400 
694- if (cops->destroy_compress_ctx)401+ cc->nr_cpages = nr_cpages;
695- cops->destroy_compress_ctx(cc);
696- 
697- cc->valid_nr_cpages = new_nr_cpages;
698 402 
699 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,403 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,
700 cc->clen, ret);404 cc->clen, ret);
701 return 0;405 return 0;
702 406 
703out_vunmap_cbuf:407out_vunmap_cbuf:
704- vm_unmap_ram(cc->cbuf, cc->nr_cpages);408+ vunmap(cc->cbuf);
705out_vunmap_rbuf:409out_vunmap_rbuf:
706- vm_unmap_ram(cc->rbuf, cc->cluster_size);410+ vunmap(cc->rbuf);
707out_free_cpages:411out_free_cpages:
708 for (i = 0; i < cc->nr_cpages; i++) {412 for (i = 0; i < cc->nr_cpages; i++) {
709 if (cc->cpages[i])413 if (cc->cpages[i])
710- f2fs_compress_free_page(cc->cpages[i]);414+ f2fs_put_compressed_page(cc->cpages[i]);
711 }415 }
712- page_array_free(cc->inode, cc->cpages, cc->nr_cpages);416+ kfree(cc->cpages);
713 cc->cpages = NULL;417 cc->cpages = NULL;
714destroy_compress_ctx:418destroy_compress_ctx:
715- if (cops->destroy_compress_ctx)419+ cops->destroy_compress_ctx(cc);
716- cops->destroy_compress_ctx(cc);
717out:420out:
718 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,421 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,
719 cc->clen, ret);422 cc->clen, ret);
720 return ret;423 return ret;
721}424}
722 425 
723-static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic,426+void f2fs_decompress_pages(struct bio *bio, struct page *page, bool verity)
724- bool pre_alloc);
725-static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic,
726- bool bypass_destroy_callback, bool pre_alloc);
727- 
728-void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task)
729{427{
428+ struct decompress_io_ctx *dic =
429+ (struct decompress_io_ctx *)page_private(page);
730 struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode);430 struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode);
731- struct f2fs_inode_info *fi = F2FS_I(dic->inode);431+ struct f2fs_inode_info *fi= F2FS_I(dic->inode);
732 const struct f2fs_compress_ops *cops =432 const struct f2fs_compress_ops *cops =
733 f2fs_cops[fi->i_compress_algorithm];433 f2fs_cops[fi->i_compress_algorithm];
734- bool bypass_callback = false;
735 int ret;434 int ret;
736 435 
436+ dec_page_count(sbi, F2FS_RD_DATA);
437+ 
438+ if (bio->bi_status || PageError(page))
439+ dic->failed = true;
440+ 
441+ if (refcount_dec_not_one(&dic->ref))
442+ return;
443+ 
737 trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx,444 trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx,
738 dic->cluster_size, fi->i_compress_algorithm);445 dic->cluster_size, fi->i_compress_algorithm);
739 446 
447+ /* submit partial compressed pages */
740 if (dic->failed) {448 if (dic->failed) {
741 ret = -EIO;449 ret = -EIO;
742- goto out_end_io;450+ goto out_free_dic;
743 }451 }
744 452 
745- ret = f2fs_prepare_decomp_mem(dic, false);453+ dic->rbuf = vmap(dic->tpages, dic->cluster_size, VM_MAP, PAGE_KERNEL);
746- if (ret) {454+ if (!dic->rbuf) {
747- bypass_callback = true;455+ ret = -ENOMEM;
748- goto out_release;456+ goto out_free_dic;
457+ }
458+ 
459+ dic->cbuf = vmap(dic->cpages, dic->nr_cpages, VM_MAP, PAGE_KERNEL_RO);
460+ if (!dic->cbuf) {
461+ ret = -ENOMEM;
462+ goto out_vunmap_rbuf;
749 }463 }
750 464 
751 dic->clen = le32_to_cpu(dic->cbuf->clen);465 dic->clen = le32_to_cpu(dic->cbuf->clen);
@@ -753,65 +467,24 @@ void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task)
753 467 
754 if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) {468 if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) {
755 ret = -EFSCORRUPTED;469 ret = -EFSCORRUPTED;
756- 470+ goto out_vunmap_cbuf;
757- /* Avoid f2fs_commit_super in irq context */
758- if (!in_task)
759- f2fs_handle_error_async(sbi, ERROR_FAIL_DECOMPRESSION);
760- else
761- f2fs_handle_error(sbi, ERROR_FAIL_DECOMPRESSION);
762- goto out_release;
763 }471 }
764 472 
765 ret = cops->decompress_pages(dic);473 ret = cops->decompress_pages(dic);
766 474 
767- if (!ret && (fi->i_compress_flag & BIT(COMPRESS_CHKSUM))) {475+out_vunmap_cbuf:
768- u32 provided = le32_to_cpu(dic->cbuf->chksum);476+ vunmap(dic->cbuf);
769- u32 calculated = f2fs_crc32(sbi, dic->cbuf->cdata, dic->clen);477+out_vunmap_rbuf:
478+ vunmap(dic->rbuf);
479+out_free_dic:
480+ if (!verity)
481+ f2fs_decompress_end_io(dic->rpages, dic->cluster_size,
482+ ret, false);
770 483 
771- if (provided != calculated) {
772- if (!is_inode_flag_set(dic->inode, FI_COMPRESS_CORRUPT)) {
773- set_inode_flag(dic->inode, FI_COMPRESS_CORRUPT);
774- f2fs_info_ratelimited(sbi,
775- "checksum invalid, nid = %lu, %x vs %x",
776- dic->inode->i_ino,
777- provided, calculated);
778- }
779- set_sbi_flag(sbi, SBI_NEED_FSCK);
780- }
781- }
782- 
783-out_release:
784- f2fs_release_decomp_mem(dic, bypass_callback, false);
785- 
786-out_end_io:
787 trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx,484 trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx,
788 dic->clen, ret);485 dic->clen, ret);
789- f2fs_decompress_end_io(dic, ret, in_task);486+ if (!verity)
790-}487+ f2fs_free_dic(dic);
791- 
792-/*
793- * This is called when a page of a compressed cluster has been read from disk
794- * (or failed to be read from disk). It checks whether this page was the last
795- * page being waited on in the cluster, and if so, it decompresses the cluster
796- * (or in the case of a failure, cleans up without actually decompressing).
797- */
798-void f2fs_end_read_compressed_page(struct page *page, bool failed,
799- block_t blkaddr, bool in_task)
800-{
801- struct decompress_io_ctx *dic =
802- (struct decompress_io_ctx *)page_private(page);
803- struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode);
804- 
805- dec_page_count(sbi, F2FS_RD_DATA);
806- 
807- if (failed)
808- WRITE_ONCE(dic->failed, true);
809- else if (blkaddr && in_task)
810- f2fs_cache_compressed_page(sbi, page,
811- dic->inode->i_ino, blkaddr);
812- 
813- if (atomic_dec_and_test(&dic->remaining_pages))
814- f2fs_decompress_cluster(dic, in_task);
815}488}
816 489 
817static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index)490static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index)
@@ -838,34 +511,9 @@ bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index)
838 return is_page_in_cluster(cc, index);511 return is_page_in_cluster(cc, index);
839}512}
840 513 
841-bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages,514+static bool __cluster_may_compress(struct compress_ctx *cc)
842- int index, int nr_pages, bool uptodate)
843-{
844- unsigned long pgidx = pages[index]->index;
845- int i = uptodate ? 0 : 1;
846- 
847- /*
848- * when uptodate set to true, try to check all pages in cluster is
849- * uptodate or not.
850- */
851- if (uptodate && (pgidx % cc->cluster_size))
852- return false;
853- 
854- if (nr_pages - index < cc->cluster_size)
855- return false;
856- 
857- for (; i < cc->cluster_size; i++) {
858- if (pages[index + i]->index != pgidx + i)
859- return false;
860- if (uptodate && !PageUptodate(pages[index + i]))
861- return false;
862- }
863- 
864- return true;
865-}
866- 
867-static bool cluster_has_invalid_data(struct compress_ctx *cc)
868{515{
516+ struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
869 loff_t i_size = i_size_read(cc->inode);517 loff_t i_size = i_size_read(cc->inode);
870 unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE);518 unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE);
871 int i;519 int i;
@@ -873,163 +521,76 @@ static bool cluster_has_invalid_data(struct compress_ctx *cc)
873 for (i = 0; i < cc->cluster_size; i++) {521 for (i = 0; i < cc->cluster_size; i++) {
874 struct page *page = cc->rpages[i];522 struct page *page = cc->rpages[i];
875 523 
876- f2fs_bug_on(F2FS_I_SB(cc->inode), !page);524+ f2fs_bug_on(sbi, !page);
525+ 
526+ if (unlikely(f2fs_cp_error(sbi)))
527+ return false;
528+ if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
529+ return false;
877 530 
878 /* beyond EOF */531 /* beyond EOF */
879 if (page->index >= nr_pages)532 if (page->index >= nr_pages)
880- return true;533+ return false;
881 }534 }
882- return false;
883-}
884- 
885-bool f2fs_sanity_check_cluster(struct dnode_of_data *dn)
886-{
887-#ifdef CONFIG_F2FS_CHECK_FS
888- struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
889- unsigned int cluster_size = F2FS_I(dn->inode)->i_cluster_size;
890- int cluster_end = 0;
891- unsigned int count;
892- int i;
893- char *reason = "";
894- 
895- if (dn->data_blkaddr != COMPRESS_ADDR)
896- return false;
897- 
898- /* [..., COMPR_ADDR, ...] */
899- if (dn->ofs_in_node % cluster_size) {
900- reason = "[*|C|*|*]";
901- goto out;
902- }
903- 
904- for (i = 1, count = 1; i < cluster_size; i++, count++) {
905- block_t blkaddr = data_blkaddr(dn->inode, dn->node_page,
906- dn->ofs_in_node + i);
907- 
908- /* [COMPR_ADDR, ..., COMPR_ADDR] */
909- if (blkaddr == COMPRESS_ADDR) {
910- reason = "[C|*|C|*]";
911- goto out;
912- }
913- if (!__is_valid_data_blkaddr(blkaddr)) {
914- if (!cluster_end)
915- cluster_end = i;
916- continue;
917- }
918- /* [COMPR_ADDR, NULL_ADDR or NEW_ADDR, valid_blkaddr] */
919- if (cluster_end) {
920- reason = "[C|N|N|V]";
921- goto out;
922- }
923- }
924- 
925- f2fs_bug_on(F2FS_I_SB(dn->inode), count != cluster_size &&
926- !is_inode_flag_set(dn->inode, FI_COMPRESS_RELEASED));
927- 
928- return false;
929-out:
930- f2fs_warn(sbi, "access invalid cluster, ino:%lu, nid:%u, ofs_in_node:%u, reason:%s",
931- dn->inode->i_ino, dn->nid, dn->ofs_in_node, reason);
932- set_sbi_flag(sbi, SBI_NEED_FSCK);
933 return true;535 return true;
934-#else
935- return false;
936-#endif
937}536}
938 537 
939-static int __f2fs_get_cluster_blocks(struct inode *inode,538+/* return # of compressed block addresses */
940- struct dnode_of_data *dn)539+static int f2fs_compressed_blocks(struct compress_ctx *cc)
941-{
942- unsigned int cluster_size = F2FS_I(inode)->i_cluster_size;
943- int count, i;
944- 
945- for (i = 0, count = 0; i < cluster_size; i++) {
946- block_t blkaddr = data_blkaddr(dn->inode, dn->node_page,
947- dn->ofs_in_node + i);
948- 
949- if (__is_valid_data_blkaddr(blkaddr))
950- count++;
951- }
952- 
953- return count;
954-}
955- 
956-static int __f2fs_cluster_blocks(struct inode *inode, unsigned int cluster_idx,
957- enum cluster_check_type type)
958{540{
959 struct dnode_of_data dn;541 struct dnode_of_data dn;
960- unsigned int start_idx = cluster_idx <<
961- F2FS_I(inode)->i_log_cluster_size;
962 int ret;542 int ret;
963 543 
964- set_new_dnode(&dn, inode, NULL, NULL, 0);544+ set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
965- ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);545+ ret = f2fs_get_dnode_of_data(&dn, start_idx_of_cluster(cc),
546+ LOOKUP_NODE);
966 if (ret) {547 if (ret) {
967 if (ret == -ENOENT)548 if (ret == -ENOENT)
968 ret = 0;549 ret = 0;
969 goto fail;550 goto fail;
970 }551 }
971 552 
972- if (f2fs_sanity_check_cluster(&dn)) {
973- ret = -EFSCORRUPTED;
974- goto fail;
975- }
976- 
977 if (dn.data_blkaddr == COMPRESS_ADDR) {553 if (dn.data_blkaddr == COMPRESS_ADDR) {
978- if (type == CLUSTER_COMPR_BLKS)554+ int i;
979- ret = 1 + __f2fs_get_cluster_blocks(inode, &dn);555+ 
980- else if (type == CLUSTER_IS_COMPR)556+ ret = 1;
981- ret = 1;557+ for (i = 1; i < cc->cluster_size; i++) {
982- } else if (type == CLUSTER_RAW_BLKS) {558+ block_t blkaddr;
983- ret = __f2fs_get_cluster_blocks(inode, &dn);559+ 
560+ blkaddr = datablock_addr(dn.inode,
561+ dn.node_page, dn.ofs_in_node + i);
562+ if (blkaddr != NULL_ADDR)
563+ ret++;
564+ }
984 }565 }
985fail:566fail:
986 f2fs_put_dnode(&dn);567 f2fs_put_dnode(&dn);
987 return ret;568 return ret;
988}569}
989 570 
990-/* return # of compressed blocks in compressed cluster */
991-static int f2fs_compressed_blocks(struct compress_ctx *cc)
992-{
993- return __f2fs_cluster_blocks(cc->inode, cc->cluster_idx,
994- CLUSTER_COMPR_BLKS);
995-}
996- 
997-/* return # of raw blocks in non-compressed cluster */
998-static int f2fs_decompressed_blocks(struct inode *inode,
999- unsigned int cluster_idx)
1000-{
1001- return __f2fs_cluster_blocks(inode, cluster_idx,
1002- CLUSTER_RAW_BLKS);
1003-}
1004- 
1005-/* return whether cluster is compressed one or not */
1006int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index)571int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index)
1007{572{
1008- return __f2fs_cluster_blocks(inode,573+ struct compress_ctx cc = {
1009- index >> F2FS_I(inode)->i_log_cluster_size,574+ .inode = inode,
1010- CLUSTER_IS_COMPR);575+ .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
1011-}576+ .cluster_size = F2FS_I(inode)->i_cluster_size,
577+ .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size,
578+ };
1012 579 
1013-/* return whether cluster contains non raw blocks or not */580+ return f2fs_compressed_blocks(&cc);
1014-bool f2fs_is_sparse_cluster(struct inode *inode, pgoff_t index)
1015-{
1016- unsigned int cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size;
1017- 
1018- return f2fs_decompressed_blocks(inode, cluster_idx) !=
1019- F2FS_I(inode)->i_cluster_size;
1020}581}
1021 582 
1022static bool cluster_may_compress(struct compress_ctx *cc)583static bool cluster_may_compress(struct compress_ctx *cc)
1023{584{
1024- if (!f2fs_need_compress_data(cc->inode))585+ if (!f2fs_compressed_file(cc->inode))
1025 return false;586 return false;
1026 if (f2fs_is_atomic_file(cc->inode))587 if (f2fs_is_atomic_file(cc->inode))
1027 return false;588 return false;
589+ if (f2fs_is_mmap_file(cc->inode))
590+ return false;
1028 if (!f2fs_cluster_is_full(cc))591 if (!f2fs_cluster_is_full(cc))
1029 return false;592 return false;
1030- if (unlikely(f2fs_cp_error(F2FS_I_SB(cc->inode))))593+ return __cluster_may_compress(cc);
1031- return false;
1032- return !cluster_has_invalid_data(cc);
1033}594}
1034 595 
1035static void set_cluster_writeback(struct compress_ctx *cc)596static void set_cluster_writeback(struct compress_ctx *cc)
@@ -1047,10 +608,8 @@ static void set_cluster_dirty(struct compress_ctx *cc)
1047 int i;608 int i;
1048 609 
1049 for (i = 0; i < cc->cluster_size; i++)610 for (i = 0; i < cc->cluster_size; i++)
1050- if (cc->rpages[i]) {611+ if (cc->rpages[i])
1051 set_page_dirty(cc->rpages[i]);612 set_page_dirty(cc->rpages[i]);
1052- set_page_private_gcing(cc->rpages[i]);
1053- }
1054}613}
1055 614 
1056static int prepare_compress_overwrite(struct compress_ctx *cc,615static int prepare_compress_overwrite(struct compress_ctx *cc,
@@ -1059,16 +618,21 @@ static int prepare_compress_overwrite(struct compress_ctx *cc,
1059 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);618 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
1060 struct address_space *mapping = cc->inode->i_mapping;619 struct address_space *mapping = cc->inode->i_mapping;
1061 struct page *page;620 struct page *page;
621+ struct dnode_of_data dn;
1062 sector_t last_block_in_bio;622 sector_t last_block_in_bio;
1063- fgf_t fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT;623+ unsigned fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT;
1064 pgoff_t start_idx = start_idx_of_cluster(cc);624 pgoff_t start_idx = start_idx_of_cluster(cc);
1065 int i, ret;625 int i, ret;
626+ bool prealloc;
1066 627 
1067retry:628retry:
1068- ret = f2fs_is_compressed_cluster(cc->inode, start_idx);629+ ret = f2fs_compressed_blocks(cc);
1069 if (ret <= 0)630 if (ret <= 0)
1070 return ret;631 return ret;
1071 632 
633+ /* compressed case */
634+ prealloc = (ret < cc->cluster_size);
635+ 
1072 ret = f2fs_init_compress_ctx(cc);636 ret = f2fs_init_compress_ctx(cc);
1073 if (ret)637 if (ret)
1074 return ret;638 return ret;
@@ -1083,7 +647,7 @@ static int prepare_compress_overwrite(struct compress_ctx *cc,
1083 }647 }
1084 648 
1085 if (PageUptodate(page))649 if (PageUptodate(page))
1086- f2fs_put_page(page, 1);650+ unlock_page(page);
1087 else651 else
1088 f2fs_compress_ctx_add_page(cc, page);652 f2fs_compress_ctx_add_page(cc, page);
1089 }653 }
@@ -1092,40 +656,57 @@ static int prepare_compress_overwrite(struct compress_ctx *cc,
1092 struct bio *bio = NULL;656 struct bio *bio = NULL;
1093 657 
1094 ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size,658 ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size,
1095- &last_block_in_bio, false, true);659+ &last_block_in_bio, false);
1096- f2fs_put_rpages(cc);660+ f2fs_destroy_compress_ctx(cc);
1097- f2fs_destroy_compress_ctx(cc, true);
1098 if (ret)661 if (ret)
1099- goto out;662+ goto release_pages;
1100 if (bio)663 if (bio)
1101- f2fs_submit_read_bio(sbi, bio, DATA);664+ f2fs_submit_bio(sbi, bio, DATA);
1102 665 
1103 ret = f2fs_init_compress_ctx(cc);666 ret = f2fs_init_compress_ctx(cc);
1104 if (ret)667 if (ret)
1105- goto out;668+ goto release_pages;
1106 }669 }
1107 670 
1108 for (i = 0; i < cc->cluster_size; i++) {671 for (i = 0; i < cc->cluster_size; i++) {
1109 f2fs_bug_on(sbi, cc->rpages[i]);672 f2fs_bug_on(sbi, cc->rpages[i]);
1110 673 
1111 page = find_lock_page(mapping, start_idx + i);674 page = find_lock_page(mapping, start_idx + i);
1112- if (!page) {675+ f2fs_bug_on(sbi, !page);
1113- /* page can be truncated */
1114- goto release_and_retry;
1115- }
1116 676 
1117 f2fs_wait_on_page_writeback(page, DATA, true, true);677 f2fs_wait_on_page_writeback(page, DATA, true, true);
678+ 
1118 f2fs_compress_ctx_add_page(cc, page);679 f2fs_compress_ctx_add_page(cc, page);
680+ f2fs_put_page(page, 0);
1119 681 
1120 if (!PageUptodate(page)) {682 if (!PageUptodate(page)) {
1121-release_and_retry:
1122- f2fs_put_rpages(cc);
1123 f2fs_unlock_rpages(cc, i + 1);683 f2fs_unlock_rpages(cc, i + 1);
1124- f2fs_destroy_compress_ctx(cc, true);684+ f2fs_put_rpages_mapping(cc, mapping, start_idx,
685+ cc->cluster_size);
686+ f2fs_destroy_compress_ctx(cc);
1125 goto retry;687 goto retry;
1126 }688 }
1127 }689 }
1128 690 
691+ if (prealloc) {
692+ __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true);
693+ 
694+ set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
695+ 
696+ for (i = cc->cluster_size - 1; i > 0; i--) {
697+ ret = f2fs_get_block(&dn, start_idx + i);
698+ if (ret) {
699+ i = cc->cluster_size;
700+ break;
701+ }
702+ 
703+ if (dn.data_blkaddr != NEW_ADDR)
704+ break;
705+ }
706+ 
707+ __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false);
708+ }
709+ 
1129 if (likely(!ret)) {710 if (likely(!ret)) {
1130 *fsdata = cc->rpages;711 *fsdata = cc->rpages;
1131 *pagep = cc->rpages[offset_in_cluster(cc, index)];712 *pagep = cc->rpages[offset_in_cluster(cc, index)];
@@ -1133,10 +714,10 @@ static int prepare_compress_overwrite(struct compress_ctx *cc,
1133 }714 }
1134 715 
1135unlock_pages:716unlock_pages:
1136- f2fs_put_rpages(cc);
1137 f2fs_unlock_rpages(cc, i);717 f2fs_unlock_rpages(cc, i);
1138- f2fs_destroy_compress_ctx(cc, true);718+release_pages:
1139-out:719+ f2fs_put_rpages_mapping(cc, mapping, start_idx, i);
720+ f2fs_destroy_compress_ctx(cc);
1140 return ret;721 return ret;
1141}722}
1142 723 
@@ -1160,7 +741,6 @@ bool f2fs_compress_write_end(struct inode *inode, void *fsdata,
1160 741 
1161{742{
1162 struct compress_ctx cc = {743 struct compress_ctx cc = {
1163- .inode = inode,
1164 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,744 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
1165 .cluster_size = F2FS_I(inode)->i_cluster_size,745 .cluster_size = F2FS_I(inode)->i_cluster_size,
1166 .rpages = fsdata,746 .rpages = fsdata,
@@ -1171,60 +751,11 @@ bool f2fs_compress_write_end(struct inode *inode, void *fsdata,
1171 set_cluster_dirty(&cc);751 set_cluster_dirty(&cc);
1172 752 
1173 f2fs_put_rpages_wbc(&cc, NULL, false, 1);753 f2fs_put_rpages_wbc(&cc, NULL, false, 1);
1174- f2fs_destroy_compress_ctx(&cc, false);754+ f2fs_destroy_compress_ctx(&cc);
1175 755 
1176 return first_index;756 return first_index;
1177}757}
1178 758 
1179-int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock)
1180-{
1181- void *fsdata = NULL;
1182- struct page *pagep;
1183- int log_cluster_size = F2FS_I(inode)->i_log_cluster_size;
1184- pgoff_t start_idx = from >> (PAGE_SHIFT + log_cluster_size) <<
1185- log_cluster_size;
1186- int err;
1187- 
1188- err = f2fs_is_compressed_cluster(inode, start_idx);
1189- if (err < 0)
1190- return err;
1191- 
1192- /* truncate normal cluster */
1193- if (!err)
1194- return f2fs_do_truncate_blocks(inode, from, lock);
1195- 
1196- /* truncate compressed cluster */
1197- err = f2fs_prepare_compress_overwrite(inode, &pagep,
1198- start_idx, &fsdata);
1199- 
1200- /* should not be a normal cluster */
1201- f2fs_bug_on(F2FS_I_SB(inode), err == 0);
1202- 
1203- if (err <= 0)
1204- return err;
1205- 
1206- if (err > 0) {
1207- struct page **rpages = fsdata;
1208- int cluster_size = F2FS_I(inode)->i_cluster_size;
1209- int i;
1210- 
1211- for (i = cluster_size - 1; i >= 0; i--) {
1212- loff_t start = rpages[i]->index << PAGE_SHIFT;
1213- 
1214- if (from <= start) {
1215- zero_user_segment(rpages[i], 0, PAGE_SIZE);
1216- } else {
1217- zero_user_segment(rpages[i], from - start,
1218- PAGE_SIZE);
1219- break;
1220- }
1221- }
1222- 
1223- f2fs_compress_write_end(inode, fsdata, start_idx, true);
1224- }
1225- return 0;
1226-}
1227- 
1228static int f2fs_write_compressed_pages(struct compress_ctx *cc,759static int f2fs_write_compressed_pages(struct compress_ctx *cc,
1229 int *submitted,760 int *submitted,
1230 struct writeback_control *wbc,761 struct writeback_control *wbc,
@@ -1243,11 +774,10 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
1243 .page = NULL,774 .page = NULL,
1244 .encrypted_page = NULL,775 .encrypted_page = NULL,
1245 .compressed_page = NULL,776 .compressed_page = NULL,
1246- .submitted = 0,777+ .submitted = false,
1247 .io_type = io_type,778 .io_type = io_type,
1248 .io_wbc = wbc,779 .io_wbc = wbc,
1249- .encrypted = fscrypt_inode_uses_fs_layer_crypto(cc->inode) ?780+ .encrypted = f2fs_encrypted_file(cc->inode),
1250- 1 : 0,
1251 };781 };
1252 struct dnode_of_data dn;782 struct dnode_of_data dn;
1253 struct node_info ni;783 struct node_info ni;
@@ -1256,24 +786,9 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
1256 unsigned int last_index = cc->cluster_size - 1;786 unsigned int last_index = cc->cluster_size - 1;
1257 loff_t psize;787 loff_t psize;
1258 int i, err;788 int i, err;
1259- bool quota_inode = IS_NOQUOTA(inode);
1260 789 
1261- /* we should bypass data pages to proceed the kworker jobs */790+ if (!f2fs_trylock_op(sbi))
1262- if (unlikely(f2fs_cp_error(sbi))) {791+ return -EAGAIN;
1263- mapping_set_error(cc->rpages[0]->mapping, -EIO);
1264- goto out_free;
1265- }
1266- 
1267- if (quota_inode) {
1268- /*
1269- * We need to wait for node_write to avoid block allocation during
1270- * checkpoint. This can only happen to quota writes which can cause
1271- * the below discard race condition.
1272- */
1273- f2fs_down_read(&sbi->node_write);
1274- } else if (!f2fs_trylock_op(sbi)) {
1275- goto out_free;
1276- }
1277 792 
1278 set_new_dnode(&dn, cc->inode, NULL, NULL, 0);793 set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
1279 794 
@@ -1282,43 +797,38 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
1282 goto out_unlock_op;797 goto out_unlock_op;
1283 798 
1284 for (i = 0; i < cc->cluster_size; i++) {799 for (i = 0; i < cc->cluster_size; i++) {
1285- if (data_blkaddr(dn.inode, dn.node_page,800+ if (datablock_addr(dn.inode, dn.node_page,
1286 dn.ofs_in_node + i) == NULL_ADDR)801 dn.ofs_in_node + i) == NULL_ADDR)
1287 goto out_put_dnode;802 goto out_put_dnode;
1288 }803 }
1289 804 
1290 psize = (loff_t)(cc->rpages[last_index]->index + 1) << PAGE_SHIFT;805 psize = (loff_t)(cc->rpages[last_index]->index + 1) << PAGE_SHIFT;
1291 806 
1292- err = f2fs_get_node_info(fio.sbi, dn.nid, &ni, false);807+ err = f2fs_get_node_info(fio.sbi, dn.nid, &ni);
1293 if (err)808 if (err)
1294 goto out_put_dnode;809 goto out_put_dnode;
1295 810 
1296 fio.version = ni.version;811 fio.version = ni.version;
1297 812 
1298- cic = f2fs_kmem_cache_alloc(cic_entry_slab, GFP_F2FS_ZERO, false, sbi);813+ cic = f2fs_kzalloc(sbi, sizeof(struct compress_io_ctx), GFP_NOFS);
1299 if (!cic)814 if (!cic)
1300 goto out_put_dnode;815 goto out_put_dnode;
1301 816 
1302 cic->magic = F2FS_COMPRESSED_PAGE_MAGIC;817 cic->magic = F2FS_COMPRESSED_PAGE_MAGIC;
1303 cic->inode = inode;818 cic->inode = inode;
1304- atomic_set(&cic->pending_pages, cc->valid_nr_cpages);819+ refcount_set(&cic->ref, 1);
1305- cic->rpages = page_array_alloc(cc->inode, cc->cluster_size);820+ cic->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
821+ cc->log_cluster_size, GFP_NOFS);
1306 if (!cic->rpages)822 if (!cic->rpages)
1307 goto out_put_cic;823 goto out_put_cic;
1308 824 
1309 cic->nr_rpages = cc->cluster_size;825 cic->nr_rpages = cc->cluster_size;
1310 826 
1311- for (i = 0; i < cc->valid_nr_cpages; i++) {827+ for (i = 0; i < cc->nr_cpages; i++) {
1312 f2fs_set_compressed_page(cc->cpages[i], inode,828 f2fs_set_compressed_page(cc->cpages[i], inode,
1313- cc->rpages[i + 1]->index, cic);829+ cc->rpages[i + 1]->index,
830+ cic, i ? &cic->ref : NULL);
1314 fio.compressed_page = cc->cpages[i];831 fio.compressed_page = cc->cpages[i];
1315- 
1316- fio.old_blkaddr = data_blkaddr(dn.inode, dn.node_page,
1317- dn.ofs_in_node + i + 1);
1318- 
1319- /* wait for GCed page writeback via META_MAPPING */
1320- f2fs_wait_on_block_writeback(inode, fio.old_blkaddr);
1321- 
1322 if (fio.encrypted) {832 if (fio.encrypted) {
1323 fio.page = cc->rpages[i + 1];833 fio.page = cc->rpages[i + 1];
1324 err = f2fs_encrypt_one_page(&fio);834 err = f2fs_encrypt_one_page(&fio);
@@ -1336,8 +846,9 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
1336 for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) {846 for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) {
1337 block_t blkaddr;847 block_t blkaddr;
1338 848 
1339- blkaddr = f2fs_data_blkaddr(&dn);849+ blkaddr = datablock_addr(dn.inode, dn.node_page,
1340- fio.page = cc->rpages[i];850+ dn.ofs_in_node);
851+ fio.page = cic->rpages[i];
1341 fio.old_blkaddr = blkaddr;852 fio.old_blkaddr = blkaddr;
1342 853 
1343 /* cluster header */854 /* cluster header */
@@ -1353,7 +864,7 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
1353 if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr))864 if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr))
1354 fio.compr_blocks++;865 fio.compr_blocks++;
1355 866 
1356- if (i > cc->valid_nr_cpages) {867+ if (i > cc->nr_cpages) {
1357 if (__is_valid_data_blkaddr(blkaddr)) {868 if (__is_valid_data_blkaddr(blkaddr)) {
1358 f2fs_invalidate_blocks(sbi, blkaddr);869 f2fs_invalidate_blocks(sbi, blkaddr);
1359 f2fs_update_data_blkaddr(&dn, NEW_ADDR);870 f2fs_update_data_blkaddr(&dn, NEW_ADDR);
@@ -1378,49 +889,40 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
1378 889 
1379 if (fio.compr_blocks)890 if (fio.compr_blocks)
1380 f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false);891 f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false);
1381- f2fs_i_compr_blocks_update(inode, cc->valid_nr_cpages, true);892+ f2fs_i_compr_blocks_update(inode, cc->nr_cpages, true);
1382- add_compr_block_stat(inode, cc->valid_nr_cpages);
1383 893 
1384 set_inode_flag(cc->inode, FI_APPEND_WRITE);894 set_inode_flag(cc->inode, FI_APPEND_WRITE);
895+ if (cc->cluster_idx == 0)
896+ set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
1385 897 
1386 f2fs_put_dnode(&dn);898 f2fs_put_dnode(&dn);
1387- if (quota_inode)899+ f2fs_unlock_op(sbi);
1388- f2fs_up_read(&sbi->node_write);
1389- else
1390- f2fs_unlock_op(sbi);
1391 900 
1392- spin_lock(&fi->i_size_lock);901+ down_write(&fi->i_sem);
1393 if (fi->last_disk_size < psize)902 if (fi->last_disk_size < psize)
1394 fi->last_disk_size = psize;903 fi->last_disk_size = psize;
1395- spin_unlock(&fi->i_size_lock);904+ up_write(&fi->i_sem);
1396 905 
1397 f2fs_put_rpages(cc);906 f2fs_put_rpages(cc);
1398- page_array_free(cc->inode, cc->cpages, cc->nr_cpages);907+ f2fs_destroy_compress_ctx(cc);
1399- cc->cpages = NULL;
1400- f2fs_destroy_compress_ctx(cc, false);
1401 return 0;908 return 0;
1402 909 
1403out_destroy_crypt:910out_destroy_crypt:
1404- page_array_free(cc->inode, cic->rpages, cc->cluster_size);911+ kfree(cic->rpages);
1405 912 
1406 for (--i; i >= 0; i--)913 for (--i; i >= 0; i--)
1407 fscrypt_finalize_bounce_page(&cc->cpages[i]);914 fscrypt_finalize_bounce_page(&cc->cpages[i]);
915+ for (i = 0; i < cc->nr_cpages; i++) {
916+ if (!cc->cpages[i])
917+ continue;
918+ f2fs_put_page(cc->cpages[i], 1);
919+ }
1408out_put_cic:920out_put_cic:
1409- kmem_cache_free(cic_entry_slab, cic);921+ kfree(cic);
1410out_put_dnode:922out_put_dnode:
1411 f2fs_put_dnode(&dn);923 f2fs_put_dnode(&dn);
1412out_unlock_op:924out_unlock_op:
1413- if (quota_inode)925+ f2fs_unlock_op(sbi);
1414- f2fs_up_read(&sbi->node_write);
1415- else
1416- f2fs_unlock_op(sbi);
1417-out_free:
1418- for (i = 0; i < cc->valid_nr_cpages; i++) {
1419- f2fs_compress_free_page(cc->cpages[i]);
1420- cc->cpages[i] = NULL;
1421- }
1422- page_array_free(cc->inode, cc->cpages, cc->nr_cpages);
1423- cc->cpages = NULL;
1424 return -EAGAIN;926 return -EAGAIN;
1425}927}
1426 928 
@@ -1429,112 +931,88 @@ void f2fs_compress_write_end_io(struct bio *bio, struct page *page)
1429 struct f2fs_sb_info *sbi = bio->bi_private;931 struct f2fs_sb_info *sbi = bio->bi_private;
1430 struct compress_io_ctx *cic =932 struct compress_io_ctx *cic =
1431 (struct compress_io_ctx *)page_private(page);933 (struct compress_io_ctx *)page_private(page);
1432- enum count_type type = WB_DATA_TYPE(page,
1433- f2fs_is_compressed_page(page));
1434 int i;934 int i;
1435 935 
1436 if (unlikely(bio->bi_status))936 if (unlikely(bio->bi_status))
1437 mapping_set_error(cic->inode->i_mapping, -EIO);937 mapping_set_error(cic->inode->i_mapping, -EIO);
1438 938 
1439- f2fs_compress_free_page(page);939+ f2fs_put_compressed_page(page);
1440 940 
1441- dec_page_count(sbi, type);941+ dec_page_count(sbi, F2FS_WB_DATA);
1442 942 
1443- if (atomic_dec_return(&cic->pending_pages))943+ if (refcount_dec_not_one(&cic->ref))
1444 return;944 return;
1445 945 
1446 for (i = 0; i < cic->nr_rpages; i++) {946 for (i = 0; i < cic->nr_rpages; i++) {
1447 WARN_ON(!cic->rpages[i]);947 WARN_ON(!cic->rpages[i]);
1448- clear_page_private_gcing(cic->rpages[i]);948+ clear_cold_data(cic->rpages[i]);
1449 end_page_writeback(cic->rpages[i]);949 end_page_writeback(cic->rpages[i]);
1450 }950 }
1451 951 
1452- page_array_free(cic->inode, cic->rpages, cic->nr_rpages);952+ kfree(cic->rpages);
1453- kmem_cache_free(cic_entry_slab, cic);953+ kfree(cic);
1454}954}
1455 955 
1456static int f2fs_write_raw_pages(struct compress_ctx *cc,956static int f2fs_write_raw_pages(struct compress_ctx *cc,
1457- int *submitted_p,957+ int *submitted,
1458 struct writeback_control *wbc,958 struct writeback_control *wbc,
1459 enum iostat_type io_type)959 enum iostat_type io_type)
1460{960{
1461 struct address_space *mapping = cc->inode->i_mapping;961 struct address_space *mapping = cc->inode->i_mapping;
1462- struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);962+ int _submitted, compr_blocks, ret;
1463- int submitted, compr_blocks, i;963+ int i = -1, err = 0;
1464- int ret = 0;
1465 964 
1466 compr_blocks = f2fs_compressed_blocks(cc);965 compr_blocks = f2fs_compressed_blocks(cc);
1467- 966+ if (compr_blocks < 0) {
1468- for (i = 0; i < cc->cluster_size; i++) {967+ err = compr_blocks;
1469- if (!cc->rpages[i])968+ goto out_err;
1470- continue;
1471- 
1472- redirty_page_for_writepage(wbc, cc->rpages[i]);
1473- unlock_page(cc->rpages[i]);
1474 }969 }
1475 970 
1476- if (compr_blocks < 0)
1477- return compr_blocks;
1478- 
1479- /* overwrite compressed cluster w/ normal cluster */
1480- if (compr_blocks > 0)
1481- f2fs_lock_op(sbi);
1482- 
1483 for (i = 0; i < cc->cluster_size; i++) {971 for (i = 0; i < cc->cluster_size; i++) {
1484 if (!cc->rpages[i])972 if (!cc->rpages[i])
1485 continue;973 continue;
1486retry_write:974retry_write:
1487- lock_page(cc->rpages[i]);
1488- 
1489 if (cc->rpages[i]->mapping != mapping) {975 if (cc->rpages[i]->mapping != mapping) {
1490-continue_unlock:
1491 unlock_page(cc->rpages[i]);976 unlock_page(cc->rpages[i]);
1492 continue;977 continue;
1493 }978 }
1494 979 
1495- if (!PageDirty(cc->rpages[i]))980+ BUG_ON(!PageLocked(cc->rpages[i]));
1496- goto continue_unlock;
1497 981 
1498- if (PageWriteback(cc->rpages[i])) {982+ ret = f2fs_write_single_data_page(cc->rpages[i], &_submitted,
1499- if (wbc->sync_mode == WB_SYNC_NONE)
1500- goto continue_unlock;
1501- f2fs_wait_on_page_writeback(cc->rpages[i], DATA, true, true);
1502- }
1503- 
1504- if (!clear_page_dirty_for_io(cc->rpages[i]))
1505- goto continue_unlock;
1506- 
1507- ret = f2fs_write_single_data_page(cc->rpages[i], &submitted,
1508 NULL, NULL, wbc, io_type,983 NULL, NULL, wbc, io_type,
1509- compr_blocks, false);984+ compr_blocks);
1510 if (ret) {985 if (ret) {
1511 if (ret == AOP_WRITEPAGE_ACTIVATE) {986 if (ret == AOP_WRITEPAGE_ACTIVATE) {
1512 unlock_page(cc->rpages[i]);987 unlock_page(cc->rpages[i]);
1513 ret = 0;988 ret = 0;
1514 } else if (ret == -EAGAIN) {989 } else if (ret == -EAGAIN) {
1515 ret = 0;990 ret = 0;
1516- /*991+ cond_resched();
1517- * for quota file, just redirty left pages to992+ congestion_wait(BLK_RW_ASYNC, HZ/50);
1518- * avoid deadlock caused by cluster update race993+ lock_page(cc->rpages[i]);
1519- * from foreground operation.994+ clear_page_dirty_for_io(cc->rpages[i]);
1520- */
1521- if (IS_NOQUOTA(cc->inode))
1522- goto out;
1523- f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
1524 goto retry_write;995 goto retry_write;
1525 }996 }
1526- goto out;997+ err = ret;
998+ goto out_fail;
1527 }999 }
1528 1000 
1529- *submitted_p += submitted;1001+ *submitted += _submitted;
1530 }1002 }
1003+ return 0;
1531 1004 
1532-out:1005+out_fail:
1533- if (compr_blocks > 0)1006+ /* TODO: revoke partially updated block addresses */
1534- f2fs_unlock_op(sbi);1007+ BUG_ON(compr_blocks);
1535- 1008+out_err:
1536- f2fs_balance_fs(sbi, true);1009+ for (++i; i < cc->cluster_size; i++) {
1537- return ret;1010+ if (!cc->rpages[i])
1011+ continue;
1012+ redirty_page_for_writepage(wbc, cc->rpages[i]);
1013+ unlock_page(cc->rpages[i]);
1014+ }
1015+ return err;
1538}1016}
1539 1017 
1540int f2fs_write_multi_pages(struct compress_ctx *cc,1018int f2fs_write_multi_pages(struct compress_ctx *cc,
@@ -1542,13 +1020,15 @@ int f2fs_write_multi_pages(struct compress_ctx *cc,
1542 struct writeback_control *wbc,1020 struct writeback_control *wbc,
1543 enum iostat_type io_type)1021 enum iostat_type io_type)
1544{1022{
1023+ struct f2fs_inode_info *fi = F2FS_I(cc->inode);
1024+ const struct f2fs_compress_ops *cops =
1025+ f2fs_cops[fi->i_compress_algorithm];
1545 int err;1026 int err;
1546 1027 
1547 *submitted = 0;1028 *submitted = 0;
1548 if (cluster_may_compress(cc)) {1029 if (cluster_may_compress(cc)) {
1549 err = f2fs_compress_pages(cc);1030 err = f2fs_compress_pages(cc);
1550 if (err == -EAGAIN) {1031 if (err == -EAGAIN) {
1551- add_compr_block_stat(cc->inode, cc->cluster_size);
1552 goto write;1032 goto write;
1553 } else if (err) {1033 } else if (err) {
1554 f2fs_put_rpages_wbc(cc, wbc, true, 1);1034 f2fs_put_rpages_wbc(cc, wbc, true, 1);
@@ -1557,6 +1037,7 @@ int f2fs_write_multi_pages(struct compress_ctx *cc,
1557 1037 
1558 err = f2fs_write_compressed_pages(cc, submitted,1038 err = f2fs_write_compressed_pages(cc, submitted,
1559 wbc, io_type);1039 wbc, io_type);
1040+ cops->destroy_compress_ctx(cc);
1560 if (!err)1041 if (!err)
1561 return 0;1042 return 0;
1562 f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN);1043 f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN);
@@ -1567,484 +1048,133 @@ int f2fs_write_multi_pages(struct compress_ctx *cc,
1567 err = f2fs_write_raw_pages(cc, submitted, wbc, io_type);1048 err = f2fs_write_raw_pages(cc, submitted, wbc, io_type);
1568 f2fs_put_rpages_wbc(cc, wbc, false, 0);1049 f2fs_put_rpages_wbc(cc, wbc, false, 0);
1569destroy_out:1050destroy_out:
1570- f2fs_destroy_compress_ctx(cc, false);1051+ f2fs_destroy_compress_ctx(cc);
1571 return err;1052 return err;
1572}1053}
1573 1054 
1574-static inline bool allow_memalloc_for_decomp(struct f2fs_sb_info *sbi,
1575- bool pre_alloc)
1576-{
1577- return pre_alloc ^ f2fs_low_mem_mode(sbi);
1578-}
1579- 
1580-static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic,
1581- bool pre_alloc)
1582-{
1583- const struct f2fs_compress_ops *cops =
1584- f2fs_cops[F2FS_I(dic->inode)->i_compress_algorithm];
1585- int i;
1586- 
1587- if (!allow_memalloc_for_decomp(F2FS_I_SB(dic->inode), pre_alloc))
1588- return 0;
1589- 
1590- dic->tpages = page_array_alloc(dic->inode, dic->cluster_size);
1591- if (!dic->tpages)
1592- return -ENOMEM;
1593- 
1594- for (i = 0; i < dic->cluster_size; i++) {
1595- if (dic->rpages[i]) {
1596- dic->tpages[i] = dic->rpages[i];
1597- continue;
1598- }
1599- 
1600- dic->tpages[i] = f2fs_compress_alloc_page();
1601- }
1602- 
1603- dic->rbuf = f2fs_vmap(dic->tpages, dic->cluster_size);
1604- if (!dic->rbuf)
1605- return -ENOMEM;
1606- 
1607- dic->cbuf = f2fs_vmap(dic->cpages, dic->nr_cpages);
1608- if (!dic->cbuf)
1609- return -ENOMEM;
1610- 
1611- if (cops->init_decompress_ctx)
1612- return cops->init_decompress_ctx(dic);
1613- 
1614- return 0;
1615-}
1616- 
1617-static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic,
1618- bool bypass_destroy_callback, bool pre_alloc)
1619-{
1620- const struct f2fs_compress_ops *cops =
1621- f2fs_cops[F2FS_I(dic->inode)->i_compress_algorithm];
1622- 
1623- if (!allow_memalloc_for_decomp(F2FS_I_SB(dic->inode), pre_alloc))
1624- return;
1625- 
1626- if (!bypass_destroy_callback && cops->destroy_decompress_ctx)
1627- cops->destroy_decompress_ctx(dic);
1628- 
1629- if (dic->cbuf)
1630- vm_unmap_ram(dic->cbuf, dic->nr_cpages);
1631- 
1632- if (dic->rbuf)
1633- vm_unmap_ram(dic->rbuf, dic->cluster_size);
1634-}
1635- 
1636-static void f2fs_free_dic(struct decompress_io_ctx *dic,
1637- bool bypass_destroy_callback);
1638- 
1639struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc)1055struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc)
1640{1056{
1057+ struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
1641 struct decompress_io_ctx *dic;1058 struct decompress_io_ctx *dic;
1642 pgoff_t start_idx = start_idx_of_cluster(cc);1059 pgoff_t start_idx = start_idx_of_cluster(cc);
1643- struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);1060+ int i;
1644- int i, ret;
1645 1061 
1646- dic = f2fs_kmem_cache_alloc(dic_entry_slab, GFP_F2FS_ZERO, false, sbi);1062+ dic = f2fs_kzalloc(sbi, sizeof(struct decompress_io_ctx), GFP_NOFS);
1647 if (!dic)1063 if (!dic)
1648 return ERR_PTR(-ENOMEM);1064 return ERR_PTR(-ENOMEM);
1649 1065 
1650- dic->rpages = page_array_alloc(cc->inode, cc->cluster_size);1066+ dic->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
1067+ cc->log_cluster_size, GFP_NOFS);
1651 if (!dic->rpages) {1068 if (!dic->rpages) {
1652- kmem_cache_free(dic_entry_slab, dic);1069+ kfree(dic);
1653 return ERR_PTR(-ENOMEM);1070 return ERR_PTR(-ENOMEM);
1654 }1071 }
1655 1072 
1656 dic->magic = F2FS_COMPRESSED_PAGE_MAGIC;1073 dic->magic = F2FS_COMPRESSED_PAGE_MAGIC;
1657 dic->inode = cc->inode;1074 dic->inode = cc->inode;
1658- atomic_set(&dic->remaining_pages, cc->nr_cpages);1075+ refcount_set(&dic->ref, 1);
1659 dic->cluster_idx = cc->cluster_idx;1076 dic->cluster_idx = cc->cluster_idx;
1660 dic->cluster_size = cc->cluster_size;1077 dic->cluster_size = cc->cluster_size;
1661 dic->log_cluster_size = cc->log_cluster_size;1078 dic->log_cluster_size = cc->log_cluster_size;
1662 dic->nr_cpages = cc->nr_cpages;1079 dic->nr_cpages = cc->nr_cpages;
1663- refcount_set(&dic->refcnt, 1);
1664 dic->failed = false;1080 dic->failed = false;
1665- dic->need_verity = f2fs_need_verity(cc->inode, start_idx);
1666 1081 
1667 for (i = 0; i < dic->cluster_size; i++)1082 for (i = 0; i < dic->cluster_size; i++)
1668 dic->rpages[i] = cc->rpages[i];1083 dic->rpages[i] = cc->rpages[i];
1669 dic->nr_rpages = cc->cluster_size;1084 dic->nr_rpages = cc->cluster_size;
1670 1085 
1671- dic->cpages = page_array_alloc(dic->inode, dic->nr_cpages);1086+ dic->cpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
1672- if (!dic->cpages) {1087+ dic->nr_cpages, GFP_NOFS);
1673- ret = -ENOMEM;1088+ if (!dic->cpages)
1674 goto out_free;1089 goto out_free;
1675- }
1676 1090 
1677 for (i = 0; i < dic->nr_cpages; i++) {1091 for (i = 0; i < dic->nr_cpages; i++) {
1678 struct page *page;1092 struct page *page;
1679 1093 
1680- page = f2fs_compress_alloc_page();1094+ page = f2fs_grab_page();
1095+ if (!page)
1096+ goto out_free;
1097+ 
1681 f2fs_set_compressed_page(page, cc->inode,1098 f2fs_set_compressed_page(page, cc->inode,
1682- start_idx + i + 1, dic);1099+ start_idx + i + 1,
1100+ dic, i ? &dic->ref : NULL);
1683 dic->cpages[i] = page;1101 dic->cpages[i] = page;
1684 }1102 }
1685 1103 
1686- ret = f2fs_prepare_decomp_mem(dic, true);1104+ dic->tpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
1687- if (ret)1105+ dic->cluster_size, GFP_NOFS);
1106+ if (!dic->tpages)
1688 goto out_free;1107 goto out_free;
1689 1108 
1109+ for (i = 0; i < dic->cluster_size; i++) {
1110+ if (cc->rpages[i])
1111+ continue;
1112+ 
1113+ dic->tpages[i] = f2fs_grab_page();
1114+ if (!dic->tpages[i])
1115+ goto out_free;
1116+ }
1117+ 
1118+ for (i = 0; i < dic->cluster_size; i++) {
1119+ if (dic->tpages[i])
1120+ continue;
1121+ dic->tpages[i] = cc->rpages[i];
1122+ }
1123+ 
1690 return dic;1124 return dic;
1691 1125 
1692out_free:1126out_free:
1693- f2fs_free_dic(dic, true);1127+ f2fs_free_dic(dic);
1694- return ERR_PTR(ret);1128+ return ERR_PTR(-ENOMEM);
1695}1129}
1696 1130 
1697-static void f2fs_free_dic(struct decompress_io_ctx *dic,1131+void f2fs_free_dic(struct decompress_io_ctx *dic)
1698- bool bypass_destroy_callback)
1699{1132{
1700 int i;1133 int i;
1701 1134 
1702- f2fs_release_decomp_mem(dic, bypass_destroy_callback, true);
1703- 
1704 if (dic->tpages) {1135 if (dic->tpages) {
1705 for (i = 0; i < dic->cluster_size; i++) {1136 for (i = 0; i < dic->cluster_size; i++) {
1706 if (dic->rpages[i])1137 if (dic->rpages[i])
1707 continue;1138 continue;
1708- if (!dic->tpages[i])1139+ unlock_page(dic->tpages[i]);
1709- continue;1140+ put_page(dic->tpages[i]);
1710- f2fs_compress_free_page(dic->tpages[i]);
1711 }1141 }
1712- page_array_free(dic->inode, dic->tpages, dic->cluster_size);1142+ kfree(dic->tpages);
1713 }1143 }
1714 1144 
1715 if (dic->cpages) {1145 if (dic->cpages) {
1716 for (i = 0; i < dic->nr_cpages; i++) {1146 for (i = 0; i < dic->nr_cpages; i++) {
1717 if (!dic->cpages[i])1147 if (!dic->cpages[i])
1718 continue;1148 continue;
1719- f2fs_compress_free_page(dic->cpages[i]);1149+ f2fs_put_compressed_page(dic->cpages[i]);
1720 }1150 }
1721- page_array_free(dic->inode, dic->cpages, dic->nr_cpages);1151+ kfree(dic->cpages);
1722 }1152 }
1723 1153 
1724- page_array_free(dic->inode, dic->rpages, dic->nr_rpages);1154+ kfree(dic->rpages);
1725- kmem_cache_free(dic_entry_slab, dic);1155+ kfree(dic);
1726}1156}
1727 1157 
1728-static void f2fs_late_free_dic(struct work_struct *work)1158+void f2fs_decompress_end_io(struct page **rpages,
1159+ unsigned int cluster_size, bool err, bool verity)
1729{1160{
1730- struct decompress_io_ctx *dic =1161+ int i;
1731- container_of(work, struct decompress_io_ctx, free_work);
1732 1162 
1733- f2fs_free_dic(dic, false);1163+ for (i = 0; i < cluster_size; i++) {
1734-}1164+ struct page *rpage = rpages[i];
1735 1165 
1736-static void f2fs_put_dic(struct decompress_io_ctx *dic, bool in_task)1166+ if (!rpage)
1737-{1167+ continue;
1738- if (refcount_dec_and_test(&dic->refcnt)) {1168+ 
1739- if (in_task) {1169+ if (err || PageError(rpage)) {
1740- f2fs_free_dic(dic, false);1170+ ClearPageUptodate(rpage);
1171+ ClearPageError(rpage);
1741 } else {1172 } else {
1742- INIT_WORK(&dic->free_work, f2fs_late_free_dic);1173+ if (!verity || fsverity_verify_page(rpage))
1743- queue_work(F2FS_I_SB(dic->inode)->post_read_wq,1174+ SetPageUptodate(rpage);
1744- &dic->free_work);1175+ else
1176+ SetPageError(rpage);
1745 }1177 }
1746- }
1747-}
1748- 
1749-static void f2fs_verify_cluster(struct work_struct *work)
1750-{
1751- struct decompress_io_ctx *dic =
1752- container_of(work, struct decompress_io_ctx, verity_work);
1753- int i;
1754- 
1755- /* Verify, update, and unlock the decompressed pages. */
1756- for (i = 0; i < dic->cluster_size; i++) {
1757- struct page *rpage = dic->rpages[i];
1758- 
1759- if (!rpage)
1760- continue;
1761- 
1762- if (fsverity_verify_page(rpage))
1763- SetPageUptodate(rpage);
1764- else
1765- ClearPageUptodate(rpage);
1766 unlock_page(rpage);1178 unlock_page(rpage);
1767 }1179 }
1768- 
1769- f2fs_put_dic(dic, true);
1770-}
1771- 
1772-/*
1773- * This is called when a compressed cluster has been decompressed
1774- * (or failed to be read and/or decompressed).
1775- */
1776-void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed,
1777- bool in_task)
1778-{
1779- int i;
1780- 
1781- if (!failed && dic->need_verity) {
1782- /*
1783- * Note that to avoid deadlocks, the verity work can't be done
1784- * on the decompression workqueue. This is because verifying
1785- * the data pages can involve reading metadata pages from the
1786- * file, and these metadata pages may be compressed.
1787- */
1788- INIT_WORK(&dic->verity_work, f2fs_verify_cluster);
1789- fsverity_enqueue_verify_work(&dic->verity_work);
1790- return;
1791- }
1792- 
1793- /* Update and unlock the cluster's pagecache pages. */
1794- for (i = 0; i < dic->cluster_size; i++) {
1795- struct page *rpage = dic->rpages[i];
1796- 
1797- if (!rpage)
1798- continue;
1799- 
1800- if (failed)
1801- ClearPageUptodate(rpage);
1802- else
1803- SetPageUptodate(rpage);
1804- unlock_page(rpage);
1805- }
1806- 
1807- /*
1808- * Release the reference to the decompress_io_ctx that was being held
1809- * for I/O completion.
1810- */
1811- f2fs_put_dic(dic, in_task);
1812-}
1813- 
1814-/*
1815- * Put a reference to a compressed page's decompress_io_ctx.
1816- *
1817- * This is called when the page is no longer needed and can be freed.
1818- */
1819-void f2fs_put_page_dic(struct page *page, bool in_task)
1820-{
1821- struct decompress_io_ctx *dic =
1822- (struct decompress_io_ctx *)page_private(page);
1823- 
1824- f2fs_put_dic(dic, in_task);
1825-}
1826- 
1827-/*
1828- * check whether cluster blocks are contiguous, and add extent cache entry
1829- * only if cluster blocks are logically and physically contiguous.
1830- */
1831-unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn,
1832- unsigned int ofs_in_node)
1833-{
1834- bool compressed = data_blkaddr(dn->inode, dn->node_page,
1835- ofs_in_node) == COMPRESS_ADDR;
1836- int i = compressed ? 1 : 0;
1837- block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_page,
1838- ofs_in_node + i);
1839- 
1840- for (i += 1; i < F2FS_I(dn->inode)->i_cluster_size; i++) {
1841- block_t blkaddr = data_blkaddr(dn->inode, dn->node_page,
1842- ofs_in_node + i);
1843- 
1844- if (!__is_valid_data_blkaddr(blkaddr))
1845- break;
1846- if (first_blkaddr + i - (compressed ? 1 : 0) != blkaddr)
1847- return 0;
1848- }
1849- 
1850- return compressed ? i - 1 : i;
1851-}
1852- 
1853-const struct address_space_operations f2fs_compress_aops = {
1854- .release_folio = f2fs_release_folio,
1855- .invalidate_folio = f2fs_invalidate_folio,
1856- .migrate_folio = filemap_migrate_folio,
1857-};
1858- 
1859-struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi)
1860-{
1861- return sbi->compress_inode->i_mapping;
1862-}
1863- 
1864-void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi, block_t blkaddr)
1865-{
1866- if (!sbi->compress_inode)
1867- return;
1868- invalidate_mapping_pages(COMPRESS_MAPPING(sbi), blkaddr, blkaddr);
1869-}
1870- 
1871-void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, struct page *page,
1872- nid_t ino, block_t blkaddr)
1873-{
1874- struct page *cpage;
1875- int ret;
1876- 
1877- if (!test_opt(sbi, COMPRESS_CACHE))
1878- return;
1879- 
1880- if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE_READ))
1881- return;
1882- 
1883- if (!f2fs_available_free_memory(sbi, COMPRESS_PAGE))
1884- return;
1885- 
1886- cpage = find_get_page(COMPRESS_MAPPING(sbi), blkaddr);
1887- if (cpage) {
1888- f2fs_put_page(cpage, 0);
1889- return;
1890- }
1891- 
1892- cpage = alloc_page(__GFP_NOWARN | __GFP_IO);
1893- if (!cpage)
1894- return;
1895- 
1896- ret = add_to_page_cache_lru(cpage, COMPRESS_MAPPING(sbi),
1897- blkaddr, GFP_NOFS);
1898- if (ret) {
1899- f2fs_put_page(cpage, 0);
1900- return;
1901- }
1902- 
1903- set_page_private_data(cpage, ino);
1904- 
1905- if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE_READ))
1906- goto out;
1907- 
1908- memcpy(page_address(cpage), page_address(page), PAGE_SIZE);
1909- SetPageUptodate(cpage);
1910-out:
1911- f2fs_put_page(cpage, 1);
1912-}
1913- 
1914-bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi, struct page *page,
1915- block_t blkaddr)
1916-{
1917- struct page *cpage;
1918- bool hitted = false;
1919- 
1920- if (!test_opt(sbi, COMPRESS_CACHE))
1921- return false;
1922- 
1923- cpage = f2fs_pagecache_get_page(COMPRESS_MAPPING(sbi),
1924- blkaddr, FGP_LOCK | FGP_NOWAIT, GFP_NOFS);
1925- if (cpage) {
1926- if (PageUptodate(cpage)) {
1927- atomic_inc(&sbi->compress_page_hit);
1928- memcpy(page_address(page),
1929- page_address(cpage), PAGE_SIZE);
1930- hitted = true;
1931- }
1932- f2fs_put_page(cpage, 1);
1933- }
1934- 
1935- return hitted;
1936-}
1937- 
1938-void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino)
1939-{
1940- struct address_space *mapping = COMPRESS_MAPPING(sbi);
1941- struct folio_batch fbatch;
1942- pgoff_t index = 0;
1943- pgoff_t end = MAX_BLKADDR(sbi);
1944- 
1945- if (!mapping->nrpages)
1946- return;
1947- 
1948- folio_batch_init(&fbatch);
1949- 
1950- do {
1951- unsigned int nr, i;
1952- 
1953- nr = filemap_get_folios(mapping, &index, end - 1, &fbatch);
1954- if (!nr)
1955- break;
1956- 
1957- for (i = 0; i < nr; i++) {
1958- struct folio *folio = fbatch.folios[i];
1959- 
1960- folio_lock(folio);
1961- if (folio->mapping != mapping) {
1962- folio_unlock(folio);
1963- continue;
1964- }
1965- 
1966- if (ino != get_page_private_data(&folio->page)) {
1967- folio_unlock(folio);
1968- continue;
1969- }
1970- 
1971- generic_error_remove_page(mapping, &folio->page);
1972- folio_unlock(folio);
1973- }
1974- folio_batch_release(&fbatch);
1975- cond_resched();
1976- } while (index < end);
1977-}
1978- 
1979-int f2fs_init_compress_inode(struct f2fs_sb_info *sbi)
1980-{
1981- struct inode *inode;
1982- 
1983- if (!test_opt(sbi, COMPRESS_CACHE))
1984- return 0;
1985- 
1986- inode = f2fs_iget(sbi->sb, F2FS_COMPRESS_INO(sbi));
1987- if (IS_ERR(inode))
1988- return PTR_ERR(inode);
1989- sbi->compress_inode = inode;
1990- 
1991- sbi->compress_percent = COMPRESS_PERCENT;
1992- sbi->compress_watermark = COMPRESS_WATERMARK;
1993- 
1994- atomic_set(&sbi->compress_page_hit, 0);
1995- 
1996- return 0;
1997-}
1998- 
1999-void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi)
2000-{
2001- if (!sbi->compress_inode)
2002- return;
2003- iput(sbi->compress_inode);
2004- sbi->compress_inode = NULL;
2005-}
2006- 
2007-int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi)
2008-{
2009- dev_t dev = sbi->sb->s_bdev->bd_dev;
2010- char slab_name[35];
2011- 
2012- if (!f2fs_sb_has_compression(sbi))
2013- return 0;
2014- 
2015- sprintf(slab_name, "f2fs_page_array_entry-%u:%u", MAJOR(dev), MINOR(dev));
2016- 
2017- sbi->page_array_slab_size = sizeof(struct page *) <<
2018- F2FS_OPTION(sbi).compress_log_size;
2019- 
2020- sbi->page_array_slab = f2fs_kmem_cache_create(slab_name,
2021- sbi->page_array_slab_size);
2022- return sbi->page_array_slab ? 0 : -ENOMEM;
2023-}
2024- 
2025-void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi)
2026-{
2027- kmem_cache_destroy(sbi->page_array_slab);
2028-}
2029- 
2030-int __init f2fs_init_compress_cache(void)
2031-{
2032- cic_entry_slab = f2fs_kmem_cache_create("f2fs_cic_entry",
2033- sizeof(struct compress_io_ctx));
2034- if (!cic_entry_slab)
2035- return -ENOMEM;
2036- dic_entry_slab = f2fs_kmem_cache_create("f2fs_dic_entry",
2037- sizeof(struct decompress_io_ctx));
2038- if (!dic_entry_slab)
2039- goto free_cic;
2040- return 0;
2041-free_cic:
2042- kmem_cache_destroy(cic_entry_slab);
2043- return -ENOMEM;
2044-}
2045- 
2046-void f2fs_destroy_compress_cache(void)
2047-{
2048- kmem_cache_destroy(dic_entry_slab);
2049- kmem_cache_destroy(cic_entry_slab);
2050}1180}
Mfs/hfsplus/bnode.c+1-0
@@ -18,6 +18,7 @@
18#include "hfsplus_fs.h"18#include "hfsplus_fs.h"
19#include "hfsplus_raw.h"19#include "hfsplus_raw.h"
20 20 
21+ 
21/* Copy a specified range of bytes from the raw data of a node */22/* Copy a specified range of bytes from the raw data of a node */
22void hfs_bnode_read(struct hfs_bnode *node, void *buf, int off, int len)23void hfs_bnode_read(struct hfs_bnode *node, void *buf, int off, int len)
23{24{
Mfs/hfsplus/btree.c+6-0
@@ -393,6 +393,12 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree)
393 len = hfs_brec_lenoff(node, 2, &off16);393 len = hfs_brec_lenoff(node, 2, &off16);
394 off = off16;394 off = off16;
395 395 
396+ if (!is_bnode_offset_valid(node, off)) {
397+ hfs_bnode_put(node);
398+ return ERR_PTR(-EIO);
399+ }
400+ len = check_and_correct_requested_length(node, off, len);
401+ 
396 off += node->page_offset;402 off += node->page_offset;
397 pagep = node->page + (off >> PAGE_SHIFT);403 pagep = node->page + (off >> PAGE_SHIFT);
398 data = kmap_local_page(*pagep);404 data = kmap_local_page(*pagep);
Mfs/hfsplus/hfsplus_fs.h+42-0
@@ -574,6 +574,48 @@ hfsplus_btree_lock_class(struct hfs_btree *tree)
574 return class;574 return class;
575}575}
576 576 
577+static inline
578+bool is_bnode_offset_valid(struct hfs_bnode *node, int off)
579+{
580+ bool is_valid = off < node->tree->node_size;
581+ 
582+ if (!is_valid) {
583+ pr_err("requested invalid offset: "
584+ "NODE: id %u, type %#x, height %u, "
585+ "node_size %u, offset %d\n",
586+ node->this, node->type, node->height,
587+ node->tree->node_size, off);
588+ }
589+ 
590+ return is_valid;
591+}
592+ 
593+static inline
594+int check_and_correct_requested_length(struct hfs_bnode *node, int off, int len)
595+{
596+ unsigned int node_size;
597+ 
598+ if (!is_bnode_offset_valid(node, off))
599+ return 0;
600+ 
601+ node_size = node->tree->node_size;
602+ 
603+ if ((off + len) > node_size) {
604+ int new_len = (int)node_size - off;
605+ 
606+ pr_err("requested length has been corrected: "
607+ "NODE: id %u, type %#x, height %u, "
608+ "node_size %u, offset %d, "
609+ "requested_len %d, corrected_len %d\n",
610+ node->this, node->type, node->height,
611+ node->tree->node_size, off, len, new_len);
612+ 
613+ return new_len;
614+ }
615+ 
616+ return len;
617+}
618+ 
577/* compatibility */619/* compatibility */
578#define hfsp_mt2ut(t) (struct timespec64){ .tv_sec = __hfsp_mt2ut(t) }620#define hfsp_mt2ut(t) (struct timespec64){ .tv_sec = __hfsp_mt2ut(t) }
579#define hfsp_ut2mt(t) __hfsp_ut2mt((t).tv_sec)621#define hfsp_ut2mt(t) __hfsp_ut2mt((t).tv_sec)
Mfs/hfsplus/inode.c+28-4
@@ -183,13 +183,29 @@ const struct dentry_operations hfsplus_dentry_operations = {
183 .d_compare = hfsplus_compare_dentry,183 .d_compare = hfsplus_compare_dentry,
184};184};
185 185 
186-static void hfsplus_get_perms(struct inode *inode,186+static int hfsplus_get_perms(struct inode *inode,
187- struct hfsplus_perm *perms, int dir)187+ struct hfsplus_perm *perms, int dir)
188{188{
189 struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb);189 struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb);
190 u16 mode;190 u16 mode;
191 191 
192 mode = be16_to_cpu(perms->mode);192 mode = be16_to_cpu(perms->mode);
193+ if (dir) {
194+ if (mode && !S_ISDIR(mode))
195+ goto bad_type;
196+ } else if (mode) {
197+ switch (mode & S_IFMT) {
198+ case S_IFREG:
199+ case S_IFLNK:
200+ case S_IFCHR:
201+ case S_IFBLK:
202+ case S_IFIFO:
203+ case S_IFSOCK:
204+ break;
205+ default:
206+ goto bad_type;
207+ }
208+ }
193 209 
194 i_uid_write(inode, be32_to_cpu(perms->owner));210 i_uid_write(inode, be32_to_cpu(perms->owner));
195 if ((test_bit(HFSPLUS_SB_UID, &sbi->flags)) || (!i_uid_read(inode) && !mode))211 if ((test_bit(HFSPLUS_SB_UID, &sbi->flags)) || (!i_uid_read(inode) && !mode))
@@ -215,6 +231,10 @@ static void hfsplus_get_perms(struct inode *inode,
215 inode->i_flags |= S_APPEND;231 inode->i_flags |= S_APPEND;
216 else232 else
217 inode->i_flags &= ~S_APPEND;233 inode->i_flags &= ~S_APPEND;
234+ return 0;
235+bad_type:
236+ pr_err("invalid file type 0%04o for inode %lu\n", mode, inode->i_ino);
237+ return -EIO;
218}238}
219 239 
220static int hfsplus_file_open(struct inode *inode, struct file *file)240static int hfsplus_file_open(struct inode *inode, struct file *file)
@@ -518,7 +538,9 @@ int hfsplus_cat_read_inode(struct inode *inode, struct hfs_find_data *fd)
518 }538 }
519 hfs_bnode_read(fd->bnode, &entry, fd->entryoffset,539 hfs_bnode_read(fd->bnode, &entry, fd->entryoffset,
520 sizeof(struct hfsplus_cat_folder));540 sizeof(struct hfsplus_cat_folder));
521- hfsplus_get_perms(inode, &folder->permissions, 1);541+ res = hfsplus_get_perms(inode, &folder->permissions, 1);
542+ if (res)
543+ goto out;
522 set_nlink(inode, 1);544 set_nlink(inode, 1);
523 inode->i_size = 2 + be32_to_cpu(folder->valence);545 inode->i_size = 2 + be32_to_cpu(folder->valence);
524 inode->i_atime = hfsp_mt2ut(folder->access_date);546 inode->i_atime = hfsp_mt2ut(folder->access_date);
@@ -546,7 +568,9 @@ int hfsplus_cat_read_inode(struct inode *inode, struct hfs_find_data *fd)
546 568 
547 hfsplus_inode_read_fork(inode, HFSPLUS_IS_RSRC(inode) ?569 hfsplus_inode_read_fork(inode, HFSPLUS_IS_RSRC(inode) ?
548 &file->rsrc_fork : &file->data_fork);570 &file->rsrc_fork : &file->data_fork);
549- hfsplus_get_perms(inode, &file->permissions, 0);571+ res = hfsplus_get_perms(inode, &file->permissions, 0);
572+ if (res)
573+ goto out;
550 set_nlink(inode, 1);574 set_nlink(inode, 1);
551 if (S_ISREG(inode->i_mode)) {575 if (S_ISREG(inode->i_mode)) {
552 if (file->permissions.dev)576 if (file->permissions.dev)
Mfs/hfsplus/super.c+18-5
@@ -67,13 +67,26 @@ struct inode *hfsplus_iget(struct super_block *sb, unsigned long ino)
67 if (!(inode->i_state & I_NEW))67 if (!(inode->i_state & I_NEW))
68 return inode;68 return inode;
69 69 
70+ atomic_set(&HFSPLUS_I(inode)->opencnt, 0);
71+ HFSPLUS_I(inode)->first_blocks = 0;
72+ HFSPLUS_I(inode)->clump_blocks = 0;
73+ HFSPLUS_I(inode)->alloc_blocks = 0;
74+ HFSPLUS_I(inode)->cached_start = U32_MAX;
75+ HFSPLUS_I(inode)->cached_blocks = 0;
76+ memset(HFSPLUS_I(inode)->first_extents, 0, sizeof(hfsplus_extent_rec));
77+ memset(HFSPLUS_I(inode)->cached_extents, 0, sizeof(hfsplus_extent_rec));
78+ HFSPLUS_I(inode)->extent_state = 0;
79+ mutex_init(&HFSPLUS_I(inode)->extents_lock);
80+ HFSPLUS_I(inode)->rsrc_inode = NULL;
81+ HFSPLUS_I(inode)->create_date = 0;
82+ HFSPLUS_I(inode)->linkid = 0;
83+ HFSPLUS_I(inode)->flags = 0;
84+ HFSPLUS_I(inode)->fs_blocks = 0;
85+ HFSPLUS_I(inode)->userflags = 0;
86+ HFSPLUS_I(inode)->subfolders = 0;
70 INIT_LIST_HEAD(&HFSPLUS_I(inode)->open_dir_list);87 INIT_LIST_HEAD(&HFSPLUS_I(inode)->open_dir_list);
71 spin_lock_init(&HFSPLUS_I(inode)->open_dir_lock);88 spin_lock_init(&HFSPLUS_I(inode)->open_dir_lock);
72- mutex_init(&HFSPLUS_I(inode)->extents_lock);89+ HFSPLUS_I(inode)->phys_size = 0;
73- HFSPLUS_I(inode)->flags = 0;
74- HFSPLUS_I(inode)->extent_state = 0;
75- HFSPLUS_I(inode)->rsrc_inode = NULL;
76- atomic_set(&HFSPLUS_I(inode)->opencnt, 0);
77 90 
78 if (inode->i_ino >= HFSPLUS_FIRSTUSER_CNID ||91 if (inode->i_ino >= HFSPLUS_FIRSTUSER_CNID ||
79 inode->i_ino == HFSPLUS_ROOT_CNID) {92 inode->i_ino == HFSPLUS_ROOT_CNID) {
Mfs/iomap/buffered-io.c+13-6
@@ -211,17 +211,24 @@ static void iomap_adjust_read_range(struct inode *inode, struct folio *folio,
211 * to avoid reading in already uptodate ranges.211 * to avoid reading in already uptodate ranges.
212 */212 */
213 if (ifs) {213 if (ifs) {
214- unsigned int i;214+ unsigned int i, blocks_skipped;
215 215 
216 /* move forward for each leading block marked uptodate */216 /* move forward for each leading block marked uptodate */
217- for (i = first; i <= last; i++) {217+ for (i = first; i <= last; i++)
218 if (!ifs_block_is_uptodate(ifs, i))218 if (!ifs_block_is_uptodate(ifs, i))
219 break;219 break;
220- *pos += block_size;220+ 
221- poff += block_size;221+ blocks_skipped = i - first;
222- plen -= block_size;222+ if (blocks_skipped) {
223- first++;223+ unsigned long block_offset = *pos & (block_size - 1);
224+ unsigned bytes_skipped =
225+ (blocks_skipped << block_bits) - block_offset;
226+ 
227+ *pos += bytes_skipped;
228+ poff += bytes_skipped;
229+ plen -= bytes_skipped;
224 }230 }
231+ first = i;
225 232 
226 /* truncate len if we find any trailing uptodate block(s) */233 /* truncate len if we find any trailing uptodate block(s) */
227 while (++i <= last) {234 while (++i <= last) {
Mfs/jfs/inode.c+1-1
@@ -145,9 +145,9 @@ void jfs_evict_inode(struct inode *inode)
145 if (!inode->i_nlink && !is_bad_inode(inode)) {145 if (!inode->i_nlink && !is_bad_inode(inode)) {
146 dquot_initialize(inode);146 dquot_initialize(inode);
147 147 
148+ truncate_inode_pages_final(&inode->i_data);
148 if (JFS_IP(inode)->fileset == FILESYSTEM_I) {149 if (JFS_IP(inode)->fileset == FILESYSTEM_I) {
149 struct inode *ipimap = JFS_SBI(inode->i_sb)->ipimap;150 struct inode *ipimap = JFS_SBI(inode->i_sb)->ipimap;
150- truncate_inode_pages_final(&inode->i_data);
151 151 
152 if (test_cflag(COMMIT_Freewmap, inode))152 if (test_cflag(COMMIT_Freewmap, inode))
153 jfs_free_zero_link(inode);153 jfs_free_zero_link(inode);
Mfs/nfs/namespace.c+6-0
@@ -149,6 +149,7 @@ struct vfsmount *nfs_d_automount(struct path *path)
149 struct vfsmount *mnt = ERR_PTR(-ENOMEM);149 struct vfsmount *mnt = ERR_PTR(-ENOMEM);
150 struct nfs_server *server = NFS_SB(path->dentry->d_sb);150 struct nfs_server *server = NFS_SB(path->dentry->d_sb);
151 struct nfs_client *client = server->nfs_client;151 struct nfs_client *client = server->nfs_client;
152+ unsigned long s_flags = path->dentry->d_sb->s_flags;
152 int timeout = READ_ONCE(nfs_mountpoint_expiry_timeout);153 int timeout = READ_ONCE(nfs_mountpoint_expiry_timeout);
153 int ret;154 int ret;
154 155 
@@ -174,6 +175,11 @@ struct vfsmount *nfs_d_automount(struct path *path)
174 fc->net_ns = get_net(client->cl_net);175 fc->net_ns = get_net(client->cl_net);
175 }176 }
176 177 
178+ /* Inherit the flags covered by NFS_SB_MASK */
179+ fc->sb_flags_mask |= NFS_SB_MASK;
180+ fc->sb_flags &= ~NFS_SB_MASK;
181+ fc->sb_flags |= s_flags & NFS_SB_MASK;
182+ 
177 /* for submounts we want the same server; referrals will reassign */183 /* for submounts we want the same server; referrals will reassign */
178 memcpy(&ctx->nfs_server._address, &client->cl_addr, client->cl_addrlen);184 memcpy(&ctx->nfs_server._address, &client->cl_addr, client->cl_addrlen);
179 ctx->nfs_server.addrlen = client->cl_addrlen;185 ctx->nfs_server.addrlen = client->cl_addrlen;
Mfs/nfs/nfs4proc.c+2-0
@@ -362,7 +362,9 @@ static void nfs4_setup_readdir(u64 cookie, __be32 *verifier, struct dentry *dent
362 *p++ = htonl(attrs); /* bitmap */362 *p++ = htonl(attrs); /* bitmap */
363 *p++ = htonl(12); /* attribute buffer length */363 *p++ = htonl(12); /* attribute buffer length */
364 *p++ = htonl(NF4DIR);364 *p++ = htonl(NF4DIR);
365+ spin_lock(&dentry->d_lock);
365 p = xdr_encode_hyper(p, NFS_FILEID(d_inode(dentry->d_parent)));366 p = xdr_encode_hyper(p, NFS_FILEID(d_inode(dentry->d_parent)));
367+ spin_unlock(&dentry->d_lock);
366 368 
367 readdir->pgbase = (char *)p - (char *)start;369 readdir->pgbase = (char *)p - (char *)start;
368 readdir->count -= readdir->pgbase;370 readdir->count -= readdir->pgbase;
Mfs/nfs/super.c+0-4
@@ -1303,10 +1303,6 @@ int nfs_get_tree_common(struct fs_context *fc)
1303 if (server->flags & NFS_MOUNT_NOAC)1303 if (server->flags & NFS_MOUNT_NOAC)
1304 fc->sb_flags |= SB_SYNCHRONOUS;1304 fc->sb_flags |= SB_SYNCHRONOUS;
1305 1305 
1306- if (ctx->clone_data.sb)
1307- if (ctx->clone_data.sb->s_flags & SB_SYNCHRONOUS)
1308- fc->sb_flags |= SB_SYNCHRONOUS;
1309- 
1310 /* Get a superblock - note that we may end up sharing one that already exists */1306 /* Get a superblock - note that we may end up sharing one that already exists */
1311 fc->s_fs_info = server;1307 fc->s_fs_info = server;
1312 s = sget_fc(fc, compare_super, nfs_set_super);1308 s = sget_fc(fc, compare_super, nfs_set_super);
Mfs/ntfs3/inode.c+1-0
@@ -1752,6 +1752,7 @@ int ntfs_link_inode(struct inode *inode, struct dentry *dentry)
1752 de = __getname();1752 de = __getname();
1753 if (!de)1753 if (!de)
1754 return -ENOMEM;1754 return -ENOMEM;
1755+ memset(de, 0, PATH_MAX);
1755 1756 
1756 /* Mark rw ntfs as dirty. It will be cleared at umount. */1757 /* Mark rw ntfs as dirty. It will be cleared at umount. */
1757 ntfs_set_state(sbi, NTFS_DIRTY_DIRTY);1758 ntfs_set_state(sbi, NTFS_DIRTY_DIRTY);
Mfs/ocfs2/move_extents.c+7-1
@@ -98,7 +98,13 @@ static int __ocfs2_move_extent(handle_t *handle,
98 98 
99 rec = &el->l_recs[index];99 rec = &el->l_recs[index];
100 100 
101- BUG_ON(ext_flags != rec->e_flags);101+ if (ext_flags != rec->e_flags) {
102+ ret = ocfs2_error(inode->i_sb,
103+ "Inode %llu has corrupted extent %d with flags 0x%x at cpos %u\n",
104+ (unsigned long long)ino, index, rec->e_flags, cpos);
105+ goto out;
106+ }
107+ 
102 /*108 /*
103 * after moving/defraging to new location, the extent is not going109 * after moving/defraging to new location, the extent is not going
104 * to be refcounted anymore.110 * to be refcounted anymore.
Mfs/ocfs2/suballoc.c+10-0
@@ -1923,6 +1923,16 @@ static int ocfs2_claim_suballoc_bits(struct ocfs2_alloc_context *ac,
1923 }1923 }
1924 1924 
1925 cl = (struct ocfs2_chain_list *) &fe->id2.i_chain;1925 cl = (struct ocfs2_chain_list *) &fe->id2.i_chain;
1926+ if (!le16_to_cpu(cl->cl_next_free_rec) ||
1927+ le16_to_cpu(cl->cl_next_free_rec) > le16_to_cpu(cl->cl_count)) {
1928+ status = ocfs2_error(ac->ac_inode->i_sb,
1929+ "Chain allocator dinode %llu has invalid next "
1930+ "free chain record %u, but only %u total\n",
1931+ (unsigned long long)le64_to_cpu(fe->i_blkno),
1932+ le16_to_cpu(cl->cl_next_free_rec),
1933+ le16_to_cpu(cl->cl_count));
1934+ goto bail;
1935+ }
1926 1936 
1927 victim = ocfs2_find_victim_chain(cl);1937 victim = ocfs2_find_victim_chain(cl);
1928 ac->ac_chain = victim;1938 ac->ac_chain = victim;
Mfs/proc/generic.c+9-3
@@ -687,6 +687,12 @@ void pde_put(struct proc_dir_entry *pde)
687 }687 }
688}688}
689 689 
690+static void pde_erase(struct proc_dir_entry *pde, struct proc_dir_entry *parent)
691+{
692+ rb_erase(&pde->subdir_node, &parent->subdir);
693+ RB_CLEAR_NODE(&pde->subdir_node);
694+}
695+ 
690/*696/*
691 * Remove a /proc entry and free it if it's not currently in use.697 * Remove a /proc entry and free it if it's not currently in use.
692 */698 */
@@ -709,7 +715,7 @@ void remove_proc_entry(const char *name, struct proc_dir_entry *parent)
709 WARN(1, "removing permanent /proc entry '%s'", de->name);715 WARN(1, "removing permanent /proc entry '%s'", de->name);
710 de = NULL;716 de = NULL;
711 } else {717 } else {
712- rb_erase(&de->subdir_node, &parent->subdir);718+ pde_erase(de, parent);
713 if (S_ISDIR(de->mode))719 if (S_ISDIR(de->mode))
714 parent->nlink--;720 parent->nlink--;
715 }721 }
@@ -753,7 +759,7 @@ int remove_proc_subtree(const char *name, struct proc_dir_entry *parent)
753 root->parent->name, root->name);759 root->parent->name, root->name);
754 return -EINVAL;760 return -EINVAL;
755 }761 }
756- rb_erase(&root->subdir_node, &parent->subdir);762+ pde_erase(root, parent);
757 763 
758 de = root;764 de = root;
759 while (1) {765 while (1) {
@@ -765,7 +771,7 @@ int remove_proc_subtree(const char *name, struct proc_dir_entry *parent)
765 next->parent->name, next->name);771 next->parent->name, next->name);
766 return -EINVAL;772 return -EINVAL;
767 }773 }
768- rb_erase(&next->subdir_node, &de->subdir);774+ pde_erase(next, de);
769 de = next;775 de = next;
770 continue;776 continue;
771 }777 }
Mfs/smb/client/fs_context.c+2-0
@@ -1363,12 +1363,14 @@ static int smb3_fs_context_parse_param(struct fs_context *fc,
1363 cifs_errorf(fc, "Unknown error parsing devname\n");1363 cifs_errorf(fc, "Unknown error parsing devname\n");
1364 goto cifs_parse_mount_err;1364 goto cifs_parse_mount_err;
1365 }1365 }
1366+ kfree(ctx->source);
1366 ctx->source = smb3_fs_context_fullpath(ctx, '/');1367 ctx->source = smb3_fs_context_fullpath(ctx, '/');
1367 if (IS_ERR(ctx->source)) {1368 if (IS_ERR(ctx->source)) {
1368 ctx->source = NULL;1369 ctx->source = NULL;
1369 cifs_errorf(fc, "OOM when copying UNC string\n");1370 cifs_errorf(fc, "OOM when copying UNC string\n");
1370 goto cifs_parse_mount_err;1371 goto cifs_parse_mount_err;
1371 }1372 }
1373+ kfree(fc->source);
1372 fc->source = kstrdup(ctx->source, GFP_KERNEL);1374 fc->source = kstrdup(ctx->source, GFP_KERNEL);
1373 if (fc->source == NULL) {1375 if (fc->source == NULL) {
1374 cifs_errorf(fc, "OOM when copying UNC string\n");1376 cifs_errorf(fc, "OOM when copying UNC string\n");
Mfs/smb/server/smb2pdu.c+2-2
@@ -2354,7 +2354,7 @@ static int smb2_set_ea(struct smb2_ea_info *eabuf, unsigned int buf_len,
2354 int rc = 0;2354 int rc = 0;
2355 unsigned int next = 0;2355 unsigned int next = 0;
2356 2356 
2357- if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength +2357+ if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength + 1 +
2358 le16_to_cpu(eabuf->EaValueLength))2358 le16_to_cpu(eabuf->EaValueLength))
2359 return -EINVAL;2359 return -EINVAL;
2360 2360 
@@ -2431,7 +2431,7 @@ static int smb2_set_ea(struct smb2_ea_info *eabuf, unsigned int buf_len,
2431 break;2431 break;
2432 }2432 }
2433 2433 
2434- if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength +2434+ if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength + 1 +
2435 le16_to_cpu(eabuf->EaValueLength)) {2435 le16_to_cpu(eabuf->EaValueLength)) {
2436 rc = -EINVAL;2436 rc = -EINVAL;
2437 break;2437 break;
Mfs/smb/server/vfs.c+4-1
@@ -333,6 +333,9 @@ static int check_lock_range(struct file *filp, loff_t start, loff_t end,
333 struct file_lock_context *ctx = locks_inode_context(file_inode(filp));333 struct file_lock_context *ctx = locks_inode_context(file_inode(filp));
334 int error = 0;334 int error = 0;
335 335 
336+ if (start == end)
337+ return 0;
338+ 
336 if (!ctx || list_empty_careful(&ctx->flc_posix))339 if (!ctx || list_empty_careful(&ctx->flc_posix))
337 return 0;340 return 0;
338 341 
@@ -833,7 +836,7 @@ int ksmbd_vfs_truncate(struct ksmbd_work *work,
833 if (size < inode->i_size) {836 if (size < inode->i_size) {
834 err = check_lock_range(filp, size,837 err = check_lock_range(filp, size,
835 inode->i_size - 1, WRITE);838 inode->i_size - 1, WRITE);
836- } else {839+ } else if (size > inode->i_size) {
837 err = check_lock_range(filp, inode->i_size,840 err = check_lock_range(filp, inode->i_size,
838 size - 1, WRITE);841 size - 1, WRITE);
839 }842 }
Minclude/linux/fbcon.h+2-0
@@ -11,6 +11,7 @@ void fbcon_suspended(struct fb_info *info);
11void fbcon_resumed(struct fb_info *info);11void fbcon_resumed(struct fb_info *info);
12int fbcon_mode_deleted(struct fb_info *info,12int fbcon_mode_deleted(struct fb_info *info,
13 struct fb_videomode *mode);13 struct fb_videomode *mode);
14+void fbcon_delete_modelist(struct list_head *head);
14void fbcon_new_modelist(struct fb_info *info);15void fbcon_new_modelist(struct fb_info *info);
15void fbcon_get_requirement(struct fb_info *info,16void fbcon_get_requirement(struct fb_info *info,
16 struct fb_blit_caps *caps);17 struct fb_blit_caps *caps);
@@ -31,6 +32,7 @@ static inline void fbcon_suspended(struct fb_info *info) {}
31static inline void fbcon_resumed(struct fb_info *info) {}32static inline void fbcon_resumed(struct fb_info *info) {}
32static inline int fbcon_mode_deleted(struct fb_info *info,33static inline int fbcon_mode_deleted(struct fb_info *info,
33 struct fb_videomode *mode) { return 0; }34 struct fb_videomode *mode) { return 0; }
35+static inline void fbcon_delete_modelist(struct list_head *head) {}
34static inline void fbcon_new_modelist(struct fb_info *info) {}36static inline void fbcon_new_modelist(struct fb_info *info) {}
35static inline void fbcon_get_requirement(struct fb_info *info,37static inline void fbcon_get_requirement(struct fb_info *info,
36 struct fb_blit_caps *caps) {}38 struct fb_blit_caps *caps) {}
Mkernel/bpf/hashtab.c+8-2
@@ -957,15 +957,21 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
957static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,957static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
958 void *value, bool onallcpus)958 void *value, bool onallcpus)
959{959{
960+ void *ptr;
961+ 
960 if (!onallcpus) {962 if (!onallcpus) {
961 /* copy true value_size bytes */963 /* copy true value_size bytes */
962- copy_map_value(&htab->map, this_cpu_ptr(pptr), value);964+ ptr = this_cpu_ptr(pptr);
965+ copy_map_value(&htab->map, ptr, value);
966+ bpf_obj_free_fields(htab->map.record, ptr);
963 } else {967 } else {
964 u32 size = round_up(htab->map.value_size, 8);968 u32 size = round_up(htab->map.value_size, 8);
965 int off = 0, cpu;969 int off = 0, cpu;
966 970 
967 for_each_possible_cpu(cpu) {971 for_each_possible_cpu(cpu) {
968- copy_map_value_long(&htab->map, per_cpu_ptr(pptr, cpu), value + off);972+ ptr = per_cpu_ptr(pptr, cpu);
973+ copy_map_value_long(&htab->map, ptr, value + off);
974+ bpf_obj_free_fields(htab->map.record, ptr);
969 off += size;975 off += size;
970 }976 }
971 }977 }
Mkernel/futex/syscalls.c+56-50
@@ -40,6 +40,56 @@ SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head,
40 return 0;40 return 0;
41}41}
42 42 
43+static inline void __user *futex_task_robust_list(struct task_struct *p, bool compat)
44+{
45+#ifdef CONFIG_COMPAT
46+ if (compat)
47+ return p->compat_robust_list;
48+#endif
49+ return p->robust_list;
50+}
51+ 
52+static void __user *futex_get_robust_list_common(int pid, bool compat)
53+{
54+ struct task_struct *p = current;
55+ void __user *head;
56+ int ret;
57+ 
58+ scoped_guard(rcu) {
59+ if (pid) {
60+ p = find_task_by_vpid(pid);
61+ if (!p)
62+ return (void __user *)ERR_PTR(-ESRCH);
63+ }
64+ get_task_struct(p);
65+ }
66+ 
67+ /*
68+ * Hold exec_update_lock to serialize with concurrent exec()
69+ * so ptrace_may_access() is checked against stable credentials
70+ */
71+ ret = down_read_killable(&p->signal->exec_update_lock);
72+ if (ret)
73+ goto err_put;
74+ 
75+ ret = -EPERM;
76+ if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS))
77+ goto err_unlock;
78+ 
79+ head = futex_task_robust_list(p, compat);
80+ 
81+ up_read(&p->signal->exec_update_lock);
82+ put_task_struct(p);
83+ 
84+ return head;
85+ 
86+err_unlock:
87+ up_read(&p->signal->exec_update_lock);
88+err_put:
89+ put_task_struct(p);
90+ return (void __user *)ERR_PTR(ret);
91+}
92+ 
43/**93/**
44 * sys_get_robust_list() - Get the robust-futex list head of a task94 * sys_get_robust_list() - Get the robust-futex list head of a task
45 * @pid: pid of the process [zero for current task]95 * @pid: pid of the process [zero for current task]
@@ -50,36 +100,14 @@ SYSCALL_DEFINE3(get_robust_list, int, pid,
50 struct robust_list_head __user * __user *, head_ptr,100 struct robust_list_head __user * __user *, head_ptr,
51 size_t __user *, len_ptr)101 size_t __user *, len_ptr)
52{102{
53- struct robust_list_head __user *head;103+ struct robust_list_head __user *head = futex_get_robust_list_common(pid, false);
54- unsigned long ret;
55- struct task_struct *p;
56 104 
57- rcu_read_lock();105+ if (IS_ERR(head))
58- 106+ return PTR_ERR(head);
59- ret = -ESRCH;
60- if (!pid)
61- p = current;
62- else {
63- p = find_task_by_vpid(pid);
64- if (!p)
65- goto err_unlock;
66- }
67- 
68- ret = -EPERM;
69- if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS))
70- goto err_unlock;
71- 
72- head = p->robust_list;
73- rcu_read_unlock();
74 107 
75 if (put_user(sizeof(*head), len_ptr))108 if (put_user(sizeof(*head), len_ptr))
76 return -EFAULT;109 return -EFAULT;
77 return put_user(head, head_ptr);110 return put_user(head, head_ptr);
78- 
79-err_unlock:
80- rcu_read_unlock();
81- 
82- return ret;
83}111}
84 112 
85long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,113long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
@@ -322,36 +350,14 @@ COMPAT_SYSCALL_DEFINE3(get_robust_list, int, pid,
322 compat_uptr_t __user *, head_ptr,350 compat_uptr_t __user *, head_ptr,
323 compat_size_t __user *, len_ptr)351 compat_size_t __user *, len_ptr)
324{352{
325- struct compat_robust_list_head __user *head;353+ struct compat_robust_list_head __user *head = futex_get_robust_list_common(pid, true);
326- unsigned long ret;
327- struct task_struct *p;
328 354 
329- rcu_read_lock();355+ if (IS_ERR(head))
330- 356+ return PTR_ERR(head);
331- ret = -ESRCH;
332- if (!pid)
333- p = current;
334- else {
335- p = find_task_by_vpid(pid);
336- if (!p)
337- goto err_unlock;
338- }
339- 
340- ret = -EPERM;
341- if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS))
342- goto err_unlock;
343- 
344- head = p->compat_robust_list;
345- rcu_read_unlock();
346 357 
347 if (put_user(sizeof(*head), len_ptr))358 if (put_user(sizeof(*head), len_ptr))
348 return -EFAULT;359 return -EFAULT;
349 return put_user(ptr_to_compat(head), head_ptr);360 return put_user(ptr_to_compat(head), head_ptr);
350- 
351-err_unlock:
352- rcu_read_unlock();
353- 
354- return ret;
355}361}
356#endif /* CONFIG_COMPAT */362#endif /* CONFIG_COMPAT */
357 363 
Mmm/kasan/report.c+1-3
@@ -385,9 +385,7 @@ static void print_address_description(void *addr, u8 tag,
385 }385 }
386 386 
387 if (is_vmalloc_addr(addr)) {387 if (is_vmalloc_addr(addr)) {
388- pr_err("The buggy address belongs to a");388+ pr_err("The buggy address %px belongs to a vmalloc virtual mapping\n", addr);
389- if (!vmalloc_dump_obj(addr))
390- pr_cont(" vmalloc virtual mapping\n");
391 page = vmalloc_to_page(addr);389 page = vmalloc_to_page(addr);
392 }390 }
393 391 
Mmm/kmemleak.c+4-1
@@ -452,6 +452,7 @@ static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
452{452{
453 unsigned long flags;453 unsigned long flags;
454 struct kmemleak_object *object;454 struct kmemleak_object *object;
455+ bool warn = false;
455 456 
456 /* try the slab allocator first */457 /* try the slab allocator first */
457 if (object_cache) {458 if (object_cache) {
@@ -469,8 +470,10 @@ static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
469 else if (mem_pool_free_count)470 else if (mem_pool_free_count)
470 object = &mem_pool[--mem_pool_free_count];471 object = &mem_pool[--mem_pool_free_count];
471 else472 else
472- pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n");473+ warn = true;
473 raw_spin_unlock_irqrestore(&kmemleak_lock, flags);474 raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
475+ if (warn)
476+ pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n");
474 477 
475 return object;478 return object;
476}479}
Mnet/bluetooth/iso.c+3-8
@@ -1216,7 +1216,7 @@ static int iso_sock_sendmsg(struct socket *sock, struct msghdr *msg,
1216 return -ENOTCONN;1216 return -ENOTCONN;
1217 }1217 }
1218 1218 
1219- mtu = iso_pi(sk)->conn->hcon->mtu;1219+ mtu = iso_pi(sk)->conn->hcon->hdev->iso_mtu;
1220 1220 
1221 release_sock(sk);1221 release_sock(sk);
1222 1222 
@@ -1314,13 +1314,8 @@ static int iso_sock_recvmsg(struct socket *sock, struct msghdr *msg,
1314 lock_sock(sk);1314 lock_sock(sk);
1315 switch (sk->sk_state) {1315 switch (sk->sk_state) {
1316 case BT_CONNECT2:1316 case BT_CONNECT2:
1317- if (test_bit(BT_SK_PA_SYNC, &pi->flags)) {1317+ iso_conn_defer_accept(pi->conn->hcon);
1318- iso_conn_big_sync(sk);1318+ sk->sk_state = BT_CONFIG;
1319- sk->sk_state = BT_LISTEN;
1320- } else {
1321- iso_conn_defer_accept(pi->conn->hcon);
1322- sk->sk_state = BT_CONFIG;
1323- }
1324 release_sock(sk);1319 release_sock(sk);
1325 return 0;1320 return 0;
1326 case BT_CONNECT:1321 case BT_CONNECT:
Mnet/bluetooth/sco.c+7-0
@@ -433,6 +433,13 @@ static void sco_sock_kill(struct sock *sk)
433 433 
434 BT_DBG("sk %p state %d", sk, sk->sk_state);434 BT_DBG("sk %p state %d", sk, sk->sk_state);
435 435 
436+ /* Sock is dead, so set conn->sk to NULL to avoid possible UAF */
437+ if (sco_pi(sk)->conn) {
438+ sco_conn_lock(sco_pi(sk)->conn);
439+ sco_pi(sk)->conn->sk = NULL;
440+ sco_conn_unlock(sco_pi(sk)->conn);
441+ }
442+ 
436 /* Kill poor orphan */443 /* Kill poor orphan */
437 bt_sock_unlink(&sco_sk_list, sk);444 bt_sock_unlink(&sco_sk_list, sk);
438 sock_set_flag(sk, SOCK_DEAD);445 sock_set_flag(sk, SOCK_DEAD);
Mnet/ceph/auth_x.c+2-0
@@ -631,6 +631,7 @@ static int handle_auth_session_key(struct ceph_auth_client *ac, u64 global_id,
631 631 
632 /* connection secret */632 /* connection secret */
633 ceph_decode_32_safe(p, end, len, e_inval);633 ceph_decode_32_safe(p, end, len, e_inval);
634+ ceph_decode_need(p, end, len, e_inval);
634 dout("%s connection secret blob len %d\n", __func__, len);635 dout("%s connection secret blob len %d\n", __func__, len);
635 if (len > 0) {636 if (len > 0) {
636 dp = *p + ceph_x_encrypt_offset();637 dp = *p + ceph_x_encrypt_offset();
@@ -648,6 +649,7 @@ static int handle_auth_session_key(struct ceph_auth_client *ac, u64 global_id,
648 649 
649 /* service tickets */650 /* service tickets */
650 ceph_decode_32_safe(p, end, len, e_inval);651 ceph_decode_32_safe(p, end, len, e_inval);
652+ ceph_decode_need(p, end, len, e_inval);
651 dout("%s service tickets blob len %d\n", __func__, len);653 dout("%s service tickets blob len %d\n", __func__, len);
652 if (len > 0) {654 if (len > 0) {
653 ret = ceph_x_proc_ticket_reply(ac, &th->session_key,655 ret = ceph_x_proc_ticket_reply(ac, &th->session_key,
Mnet/ceph/messenger_v2.c+7-4
@@ -1091,13 +1091,16 @@ static int decrypt_control_remainder(struct ceph_connection *con)
1091static int process_v2_sparse_read(struct ceph_connection *con,1091static int process_v2_sparse_read(struct ceph_connection *con,
1092 struct page **pages, int spos)1092 struct page **pages, int spos)
1093{1093{
1094- struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor;1094+ struct ceph_msg_data_cursor cursor;
1095 int ret;1095 int ret;
1096 1096 
1097+ ceph_msg_data_cursor_init(&cursor, con->in_msg,
1098+ con->in_msg->sparse_read_total);
1099+ 
1097 for (;;) {1100 for (;;) {
1098 char *buf = NULL;1101 char *buf = NULL;
1099 1102 
1100- ret = con->ops->sparse_read(con, cursor, &buf);1103+ ret = con->ops->sparse_read(con, &cursor, &buf);
1101 if (ret <= 0)1104 if (ret <= 0)
1102 return ret;1105 return ret;
1103 1106 
@@ -1115,11 +1118,11 @@ static int process_v2_sparse_read(struct ceph_connection *con,
1115 } else {1118 } else {
1116 struct bio_vec bv;1119 struct bio_vec bv;
1117 1120 
1118- get_bvec_at(cursor, &bv);1121+ get_bvec_at(&cursor, &bv);
1119 len = min_t(int, len, bv.bv_len);1122 len = min_t(int, len, bv.bv_len);
1120 memcpy_page(bv.bv_page, bv.bv_offset,1123 memcpy_page(bv.bv_page, bv.bv_offset,
1121 spage, soff, len);1124 spage, soff, len);
1122- ceph_msg_data_advance(cursor, len);1125+ ceph_msg_data_advance(&cursor, len);
1123 }1126 }
1124 spos += len;1127 spos += len;
1125 ret -= len;1128 ret -= len;
Mnet/ipv4/udp_tunnel_nic.c+1-1
@@ -899,7 +899,7 @@ udp_tunnel_nic_netdevice_event(struct notifier_block *unused,
899 899 
900 err = udp_tunnel_nic_register(dev);900 err = udp_tunnel_nic_register(dev);
901 if (err)901 if (err)
902- netdev_WARN(dev, "failed to register for UDP tunnel offloads: %d", err);902+ netdev_warn(dev, "failed to register for UDP tunnel offloads: %d", err);
903 return notifier_from_errno(err);903 return notifier_from_errno(err);
904 }904 }
905 /* All other events will need the udp_tunnel_nic state */905 /* All other events will need the udp_tunnel_nic state */
Mnet/ipv6/ah6.c+31-19
@@ -46,6 +46,34 @@ struct ah_skb_cb {
46 46 
47#define AH_SKB_CB(__skb) ((struct ah_skb_cb *)&((__skb)->cb[0]))47#define AH_SKB_CB(__skb) ((struct ah_skb_cb *)&((__skb)->cb[0]))
48 48 
49+/* Helper to save IPv6 addresses and extension headers to temporary storage */
50+static inline void ah6_save_hdrs(struct tmp_ext *iph_ext,
51+ struct ipv6hdr *top_iph, int extlen)
52+{
53+ if (!extlen)
54+ return;
55+ 
56+#if IS_ENABLED(CONFIG_IPV6_MIP6)
57+ iph_ext->saddr = top_iph->saddr;
58+#endif
59+ iph_ext->daddr = top_iph->daddr;
60+ memcpy(&iph_ext->hdrs, top_iph + 1, extlen - sizeof(*iph_ext));
61+}
62+ 
63+/* Helper to restore IPv6 addresses and extension headers from temporary storage */
64+static inline void ah6_restore_hdrs(struct ipv6hdr *top_iph,
65+ struct tmp_ext *iph_ext, int extlen)
66+{
67+ if (!extlen)
68+ return;
69+ 
70+#if IS_ENABLED(CONFIG_IPV6_MIP6)
71+ top_iph->saddr = iph_ext->saddr;
72+#endif
73+ top_iph->daddr = iph_ext->daddr;
74+ memcpy(top_iph + 1, &iph_ext->hdrs, extlen - sizeof(*iph_ext));
75+}
76+ 
49static void *ah_alloc_tmp(struct crypto_ahash *ahash, int nfrags,77static void *ah_alloc_tmp(struct crypto_ahash *ahash, int nfrags,
50 unsigned int size)78 unsigned int size)
51{79{
@@ -304,13 +332,7 @@ static void ah6_output_done(void *data, int err)
304 memcpy(ah->auth_data, icv, ahp->icv_trunc_len);332 memcpy(ah->auth_data, icv, ahp->icv_trunc_len);
305 memcpy(top_iph, iph_base, IPV6HDR_BASELEN);333 memcpy(top_iph, iph_base, IPV6HDR_BASELEN);
306 334 
307- if (extlen) {335+ ah6_restore_hdrs(top_iph, iph_ext, extlen);
308-#if IS_ENABLED(CONFIG_IPV6_MIP6)
309- memcpy(&top_iph->saddr, iph_ext, extlen);
310-#else
311- memcpy(&top_iph->daddr, iph_ext, extlen);
312-#endif
313- }
314 336 
315 kfree(AH_SKB_CB(skb)->tmp);337 kfree(AH_SKB_CB(skb)->tmp);
316 xfrm_output_resume(skb->sk, skb, err);338 xfrm_output_resume(skb->sk, skb, err);
@@ -381,12 +403,8 @@ static int ah6_output(struct xfrm_state *x, struct sk_buff *skb)
381 */403 */
382 memcpy(iph_base, top_iph, IPV6HDR_BASELEN);404 memcpy(iph_base, top_iph, IPV6HDR_BASELEN);
383 405 
406+ ah6_save_hdrs(iph_ext, top_iph, extlen);
384 if (extlen) {407 if (extlen) {
385-#if IS_ENABLED(CONFIG_IPV6_MIP6)
386- memcpy(iph_ext, &top_iph->saddr, extlen);
387-#else
388- memcpy(iph_ext, &top_iph->daddr, extlen);
389-#endif
390 err = ipv6_clear_mutable_options(top_iph,408 err = ipv6_clear_mutable_options(top_iph,
391 extlen - sizeof(*iph_ext) +409 extlen - sizeof(*iph_ext) +
392 sizeof(*top_iph),410 sizeof(*top_iph),
@@ -437,13 +455,7 @@ static int ah6_output(struct xfrm_state *x, struct sk_buff *skb)
437 memcpy(ah->auth_data, icv, ahp->icv_trunc_len);455 memcpy(ah->auth_data, icv, ahp->icv_trunc_len);
438 memcpy(top_iph, iph_base, IPV6HDR_BASELEN);456 memcpy(top_iph, iph_base, IPV6HDR_BASELEN);
439 457 
440- if (extlen) {458+ ah6_restore_hdrs(top_iph, iph_ext, extlen);
441-#if IS_ENABLED(CONFIG_IPV6_MIP6)
442- memcpy(&top_iph->saddr, iph_ext, extlen);
443-#else
444- memcpy(&top_iph->daddr, iph_ext, extlen);
445-#endif
446- }
447 459 
448out_free:460out_free:
449 kfree(iph_base);461 kfree(iph_base);
Mnet/ipv6/calipso.c+2-1
@@ -1345,7 +1345,8 @@ static int calipso_skbuff_setattr(struct sk_buff *skb,
1345 /* At this point new_end aligns to 4n, so (new_end & 4) pads to 8n */1345 /* At this point new_end aligns to 4n, so (new_end & 4) pads to 8n */
1346 pad = ((new_end & 4) + (end & 7)) & 7;1346 pad = ((new_end & 4) + (end & 7)) & 7;
1347 len_delta = new_end - (int)end + pad;1347 len_delta = new_end - (int)end + pad;
1348- ret_val = skb_cow(skb, skb_headroom(skb) + len_delta);1348+ ret_val = skb_cow(skb,
1349+ skb_headroom(skb) + (len_delta > 0 ? len_delta : 0));
1349 if (ret_val < 0)1350 if (ret_val < 0)
1350 return ret_val;1351 return ret_val;
1351 1352 
Mnet/ipv6/ip6_gre.c+8-1
@@ -1382,9 +1382,16 @@ static int ip6gre_header(struct sk_buff *skb, struct net_device *dev,
1382{1382{
1383 struct ip6_tnl *t = netdev_priv(dev);1383 struct ip6_tnl *t = netdev_priv(dev);
1384 struct ipv6hdr *ipv6h;1384 struct ipv6hdr *ipv6h;
1385+ int needed;
1385 __be16 *p;1386 __be16 *p;
1386 1387 
1387- ipv6h = skb_push(skb, t->hlen + sizeof(*ipv6h));1388+ needed = t->hlen + sizeof(*ipv6h);
1389+ if (skb_headroom(skb) < needed &&
1390+ pskb_expand_head(skb, HH_DATA_ALIGN(needed - skb_headroom(skb)),
1391+ 0, GFP_ATOMIC))
1392+ return -needed;
1393+ 
1394+ ipv6h = skb_push(skb, needed);
1388 ip6_flow_hdr(ipv6h, 0, ip6_make_flowlabel(dev_net(dev), skb,1395 ip6_flow_hdr(ipv6h, 0, ip6_make_flowlabel(dev_net(dev), skb,
1389 t->fl.u.ip6.flowlabel,1396 t->fl.u.ip6.flowlabel,
1390 true, &t->fl.u.ip6));1397 true, &t->fl.u.ip6));
Mnet/mptcp/options.c+10-0
@@ -408,6 +408,16 @@ bool mptcp_syn_options(struct sock *sk, const struct sk_buff *skb,
408 */408 */
409 subflow->snd_isn = TCP_SKB_CB(skb)->end_seq;409 subflow->snd_isn = TCP_SKB_CB(skb)->end_seq;
410 if (subflow->request_mptcp) {410 if (subflow->request_mptcp) {
411+ if (unlikely(subflow_simultaneous_connect(sk))) {
412+ WARN_ON_ONCE(!mptcp_try_fallback(sk, MPTCP_MIB_SIMULTCONNFALLBACK));
413+ 
414+ /* Ensure mptcp_finish_connect() will not process the
415+ * MPC handshake.
416+ */
417+ subflow->request_mptcp = 0;
418+ return false;
419+ }
420+ 
411 opts->suboptions = OPTION_MPTCP_MPC_SYN;421 opts->suboptions = OPTION_MPTCP_MPC_SYN;
412 opts->csum_reqd = mptcp_is_checksum_enabled(sock_net(sk));422 opts->csum_reqd = mptcp_is_checksum_enabled(sock_net(sk));
413 opts->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk));423 opts->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk));
Mnet/mptcp/subflow.c+211-138
@@ -9,19 +9,18 @@
9#include <linux/kernel.h>9#include <linux/kernel.h>
10#include <linux/module.h>10#include <linux/module.h>
11#include <linux/netdevice.h>11#include <linux/netdevice.h>
12-#include <crypto/algapi.h>
13#include <crypto/sha2.h>12#include <crypto/sha2.h>
13+#include <crypto/utils.h>
14#include <net/sock.h>14#include <net/sock.h>
15#include <net/inet_common.h>15#include <net/inet_common.h>
16#include <net/inet_hashtables.h>16#include <net/inet_hashtables.h>
17#include <net/protocol.h>17#include <net/protocol.h>
18-#include <net/tcp.h>
19#if IS_ENABLED(CONFIG_MPTCP_IPV6)18#if IS_ENABLED(CONFIG_MPTCP_IPV6)
20#include <net/ip6_route.h>19#include <net/ip6_route.h>
21#include <net/transp_v6.h>20#include <net/transp_v6.h>
22#endif21#endif
23#include <net/mptcp.h>22#include <net/mptcp.h>
24-#include <uapi/linux/mptcp.h>23+ 
25#include "protocol.h"24#include "protocol.h"
26#include "mib.h"25#include "mib.h"
27 26 
@@ -75,7 +74,8 @@ static void subflow_req_create_thmac(struct mptcp_subflow_request_sock *subflow_
75 74 
76 get_random_bytes(&subflow_req->local_nonce, sizeof(u32));75 get_random_bytes(&subflow_req->local_nonce, sizeof(u32));
77 76 
78- subflow_generate_hmac(msk->local_key, msk->remote_key,77+ subflow_generate_hmac(READ_ONCE(msk->local_key),
78+ READ_ONCE(msk->remote_key),
79 subflow_req->local_nonce,79 subflow_req->local_nonce,
80 subflow_req->remote_nonce, hmac);80 subflow_req->remote_nonce, hmac);
81 81 
@@ -159,8 +159,10 @@ static int subflow_check_req(struct request_sock *req,
159 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of159 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of
160 * TCP option space.160 * TCP option space.
161 */161 */
162- if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info))162+ if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info)) {
163+ subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP);
163 return -EINVAL;164 return -EINVAL;
165+ }
164#endif166#endif
165 167 
166 mptcp_get_options(skb, &mp_opt);168 mptcp_get_options(skb, &mp_opt);
@@ -235,6 +237,7 @@ static int subflow_check_req(struct request_sock *req,
235 ntohs(inet_sk((struct sock *)subflow_req->msk)->inet_sport));237 ntohs(inet_sk((struct sock *)subflow_req->msk)->inet_sport));
236 if (!mptcp_pm_sport_in_anno_list(subflow_req->msk, sk_listener)) {238 if (!mptcp_pm_sport_in_anno_list(subflow_req->msk, sk_listener)) {
237 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTSYNRX);239 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTSYNRX);
240+ subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
238 return -EPERM;241 return -EPERM;
239 }242 }
240 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTSYNRX);243 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTSYNRX);
@@ -243,10 +246,13 @@ static int subflow_check_req(struct request_sock *req,
243 subflow_req_create_thmac(subflow_req);246 subflow_req_create_thmac(subflow_req);
244 247 
245 if (unlikely(req->syncookie)) {248 if (unlikely(req->syncookie)) {
246- if (mptcp_can_accept_new_subflow(subflow_req->msk))249+ if (!mptcp_can_accept_new_subflow(subflow_req->msk)) {
247- subflow_init_req_cookie_join_save(subflow_req, skb);250+ SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINREJECTED);
248- else251+ subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
249 return -EPERM;252 return -EPERM;
253+ }
254+ 
255+ subflow_init_req_cookie_join_save(subflow_req, skb);
250 }256 }
251 257 
252 pr_debug("token=%u, remote_nonce=%u msk=%p\n", subflow_req->token,258 pr_debug("token=%u, remote_nonce=%u msk=%p\n", subflow_req->token,
@@ -297,10 +303,21 @@ int mptcp_subflow_init_cookie_req(struct request_sock *req,
297}303}
298EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req);304EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req);
299 305 
306+static enum sk_rst_reason mptcp_get_rst_reason(const struct sk_buff *skb)
307+{
308+ const struct mptcp_ext *mpext = mptcp_get_ext(skb);
309+ 
310+ if (!mpext)
311+ return SK_RST_REASON_NOT_SPECIFIED;
312+ 
313+ return sk_rst_convert_mptcp_reason(mpext->reset_reason);
314+}
315+ 
300static struct dst_entry *subflow_v4_route_req(const struct sock *sk,316static struct dst_entry *subflow_v4_route_req(const struct sock *sk,
301 struct sk_buff *skb,317 struct sk_buff *skb,
302 struct flowi *fl,318 struct flowi *fl,
303- struct request_sock *req)319+ struct request_sock *req,
320+ u32 tw_isn)
304{321{
305 struct dst_entry *dst;322 struct dst_entry *dst;
306 int err;323 int err;
@@ -308,7 +325,7 @@ static struct dst_entry *subflow_v4_route_req(const struct sock *sk,
308 tcp_rsk(req)->is_mptcp = 1;325 tcp_rsk(req)->is_mptcp = 1;
309 subflow_init_req(req, sk);326 subflow_init_req(req, sk);
310 327 
311- dst = tcp_request_sock_ipv4_ops.route_req(sk, skb, fl, req);328+ dst = tcp_request_sock_ipv4_ops.route_req(sk, skb, fl, req, tw_isn);
312 if (!dst)329 if (!dst)
313 return NULL;330 return NULL;
314 331 
@@ -318,7 +335,8 @@ static struct dst_entry *subflow_v4_route_req(const struct sock *sk,
318 335 
319 dst_release(dst);336 dst_release(dst);
320 if (!req->syncookie)337 if (!req->syncookie)
321- tcp_request_sock_ops.send_reset(sk, skb);338+ tcp_request_sock_ops.send_reset(sk, skb,
339+ mptcp_get_rst_reason(skb));
322 return NULL;340 return NULL;
323}341}
324 342 
@@ -367,7 +385,8 @@ static int subflow_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
367static struct dst_entry *subflow_v6_route_req(const struct sock *sk,385static struct dst_entry *subflow_v6_route_req(const struct sock *sk,
368 struct sk_buff *skb,386 struct sk_buff *skb,
369 struct flowi *fl,387 struct flowi *fl,
370- struct request_sock *req)388+ struct request_sock *req,
389+ u32 tw_isn)
371{390{
372 struct dst_entry *dst;391 struct dst_entry *dst;
373 int err;392 int err;
@@ -375,7 +394,7 @@ static struct dst_entry *subflow_v6_route_req(const struct sock *sk,
375 tcp_rsk(req)->is_mptcp = 1;394 tcp_rsk(req)->is_mptcp = 1;
376 subflow_init_req(req, sk);395 subflow_init_req(req, sk);
377 396 
378- dst = tcp_request_sock_ipv6_ops.route_req(sk, skb, fl, req);397+ dst = tcp_request_sock_ipv6_ops.route_req(sk, skb, fl, req, tw_isn);
379 if (!dst)398 if (!dst)
380 return NULL;399 return NULL;
381 400 
@@ -385,7 +404,8 @@ static struct dst_entry *subflow_v6_route_req(const struct sock *sk,
385 404 
386 dst_release(dst);405 dst_release(dst);
387 if (!req->syncookie)406 if (!req->syncookie)
388- tcp6_request_sock_ops.send_reset(sk, skb);407+ tcp6_request_sock_ops.send_reset(sk, skb,
408+ mptcp_get_rst_reason(skb));
389 return NULL;409 return NULL;
390}410}
391#endif411#endif
@@ -421,7 +441,7 @@ void mptcp_subflow_reset(struct sock *ssk)
421 /* must hold: tcp_done() could drop last reference on parent */441 /* must hold: tcp_done() could drop last reference on parent */
422 sock_hold(sk);442 sock_hold(sk);
423 443 
424- tcp_send_active_reset(ssk, GFP_ATOMIC);444+ mptcp_send_active_reset_reason(ssk);
425 tcp_done(ssk);445 tcp_done(ssk);
426 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags))446 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags))
427 mptcp_schedule_work(sk);447 mptcp_schedule_work(sk);
@@ -471,6 +491,9 @@ static void subflow_set_remote_key(struct mptcp_sock *msk,
471 mptcp_crypto_key_sha(subflow->remote_key, NULL, &subflow->iasn);491 mptcp_crypto_key_sha(subflow->remote_key, NULL, &subflow->iasn);
472 subflow->iasn++;492 subflow->iasn++;
473 493 
494+ /* for fallback's sake */
495+ subflow->map_seq = subflow->iasn;
496+ 
474 WRITE_ONCE(msk->remote_key, subflow->remote_key);497 WRITE_ONCE(msk->remote_key, subflow->remote_key);
475 WRITE_ONCE(msk->ack_seq, subflow->iasn);498 WRITE_ONCE(msk->ack_seq, subflow->iasn);
476 WRITE_ONCE(msk->can_ack, true);499 WRITE_ONCE(msk->can_ack, true);
@@ -524,12 +547,13 @@ static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb)
524 mptcp_get_options(skb, &mp_opt);547 mptcp_get_options(skb, &mp_opt);
525 if (subflow->request_mptcp) {548 if (subflow->request_mptcp) {
526 if (!(mp_opt.suboptions & OPTION_MPTCP_MPC_SYNACK)) {549 if (!(mp_opt.suboptions & OPTION_MPTCP_MPC_SYNACK)) {
527- if (!mptcp_try_fallback(sk))550+ if (!mptcp_try_fallback(sk,
551+ MPTCP_MIB_MPCAPABLEACTIVEFALLBACK)) {
552+ MPTCP_INC_STATS(sock_net(sk),
553+ MPTCP_MIB_FALLBACKFAILED);
528 goto do_reset;554 goto do_reset;
555+ }
529 556 
530- MPTCP_INC_STATS(sock_net(sk),
531- MPTCP_MIB_MPCAPABLEACTIVEFALLBACK);
532- pr_fallback(msk);
533 goto fallback;557 goto fallback;
534 }558 }
535 559 
@@ -540,6 +564,7 @@ static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb)
540 subflow->mp_capable = 1;564 subflow->mp_capable = 1;
541 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVEACK);565 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVEACK);
542 mptcp_finish_connect(sk);566 mptcp_finish_connect(sk);
567+ mptcp_active_enable(parent);
543 mptcp_propagate_state(parent, sk, subflow, &mp_opt);568 mptcp_propagate_state(parent, sk, subflow, &mp_opt);
544 } else if (subflow->request_join) {569 } else if (subflow->request_join) {
545 u8 hmac[SHA256_DIGEST_SIZE];570 u8 hmac[SHA256_DIGEST_SIZE];
@@ -585,6 +610,9 @@ static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb)
585 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINPORTSYNACKRX);610 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINPORTSYNACKRX);
586 }611 }
587 } else if (mptcp_check_fallback(sk)) {612 } else if (mptcp_check_fallback(sk)) {
613+ /* It looks like MPTCP is blocked, while TCP is not */
614+ if (subflow->mpc_drop)
615+ mptcp_active_disable(parent);
588fallback:616fallback:
589 mptcp_propagate_state(parent, sk, subflow, NULL);617 mptcp_propagate_state(parent, sk, subflow, NULL);
590 }618 }
@@ -724,17 +752,14 @@ struct request_sock *mptcp_subflow_reqsk_alloc(const struct request_sock_ops *op
724EXPORT_SYMBOL(mptcp_subflow_reqsk_alloc);752EXPORT_SYMBOL(mptcp_subflow_reqsk_alloc);
725 753 
726/* validate hmac received in third ACK */754/* validate hmac received in third ACK */
727-static bool subflow_hmac_valid(const struct request_sock *req,755+static bool subflow_hmac_valid(const struct mptcp_subflow_request_sock *subflow_req,
728 const struct mptcp_options_received *mp_opt)756 const struct mptcp_options_received *mp_opt)
729{757{
730- const struct mptcp_subflow_request_sock *subflow_req;758+ struct mptcp_sock *msk = subflow_req->msk;
731 u8 hmac[SHA256_DIGEST_SIZE];759 u8 hmac[SHA256_DIGEST_SIZE];
732- struct mptcp_sock *msk;
733 760 
734- subflow_req = mptcp_subflow_rsk(req);761+ subflow_generate_hmac(READ_ONCE(msk->remote_key),
735- msk = subflow_req->msk;762+ READ_ONCE(msk->local_key),
736- 
737- subflow_generate_hmac(msk->remote_key, msk->local_key,
738 subflow_req->remote_nonce,763 subflow_req->remote_nonce,
739 subflow_req->local_nonce, hmac);764 subflow_req->local_nonce, hmac);
740 765 
@@ -776,11 +801,8 @@ void __mptcp_subflow_fully_established(struct mptcp_sock *msk,
776 const struct mptcp_options_received *mp_opt)801 const struct mptcp_options_received *mp_opt)
777{802{
778 subflow_set_remote_key(msk, subflow, mp_opt);803 subflow_set_remote_key(msk, subflow, mp_opt);
779- subflow->fully_established = 1;804+ WRITE_ONCE(subflow->fully_established, true);
780 WRITE_ONCE(msk->fully_established, true);805 WRITE_ONCE(msk->fully_established, true);
781- 
782- if (subflow->is_mptfo)
783- __mptcp_fastopen_gen_msk_ackseq(msk, subflow, mp_opt);
784}806}
785 807 
786static struct sock *subflow_syn_recv_sock(const struct sock *sk,808static struct sock *subflow_syn_recv_sock(const struct sock *sk,
@@ -794,6 +816,7 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk,
794 struct mptcp_subflow_request_sock *subflow_req;816 struct mptcp_subflow_request_sock *subflow_req;
795 struct mptcp_options_received mp_opt;817 struct mptcp_options_received mp_opt;
796 bool fallback, fallback_is_fatal;818 bool fallback, fallback_is_fatal;
819+ enum sk_rst_reason reason;
797 struct mptcp_sock *owner;820 struct mptcp_sock *owner;
798 struct sock *child;821 struct sock *child;
799 822 
@@ -863,6 +886,10 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk,
863 886 
864 ctx->subflow_id = 1;887 ctx->subflow_id = 1;
865 owner = mptcp_sk(ctx->conn);888 owner = mptcp_sk(ctx->conn);
889+ 
890+ if (mp_opt.deny_join_id0)
891+ WRITE_ONCE(owner->pm.remote_deny_join_id0, true);
892+ 
866 mptcp_pm_new_connection(owner, child, 1);893 mptcp_pm_new_connection(owner, child, 1);
867 894 
868 /* with OoO packets we can reach here without ingress895 /* with OoO packets we can reach here without ingress
@@ -879,13 +906,14 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk,
879 goto dispose_child;906 goto dispose_child;
880 }907 }
881 908 
882- if (!subflow_hmac_valid(req, &mp_opt)) {909+ if (!subflow_hmac_valid(subflow_req, &mp_opt)) {
883 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC);910 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC);
884 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);911 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
885 goto dispose_child;912 goto dispose_child;
886 }913 }
887 914 
888 if (!mptcp_can_accept_new_subflow(owner)) {915 if (!mptcp_can_accept_new_subflow(owner)) {
916+ SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINREJECTED);
889 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);917 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
890 goto dispose_child;918 goto dispose_child;
891 }919 }
@@ -900,13 +928,18 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk,
900 ntohs(inet_sk((struct sock *)owner)->inet_sport));928 ntohs(inet_sk((struct sock *)owner)->inet_sport));
901 if (!mptcp_pm_sport_in_anno_list(owner, sk)) {929 if (!mptcp_pm_sport_in_anno_list(owner, sk)) {
902 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTACKRX);930 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTACKRX);
931+ subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
903 goto dispose_child;932 goto dispose_child;
904 }933 }
905 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTACKRX);934 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTACKRX);
906 }935 }
907 936 
908- if (!mptcp_finish_join(child))937+ if (!mptcp_finish_join(child)) {
938+ struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(child);
939+ 
940+ subflow_add_reset_reason(skb, subflow->reset_reason);
909 goto dispose_child;941 goto dispose_child;
942+ }
910 943 
911 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX);944 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX);
912 tcp_rsk(req)->drop_req = true;945 tcp_rsk(req)->drop_req = true;
@@ -914,7 +947,7 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk,
914 }947 }
915 948 
916 /* check for expected invariant - should never trigger, just help949 /* check for expected invariant - should never trigger, just help
917- * catching eariler subtle bugs950+ * catching earlier subtle bugs
918 */951 */
919 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp &&952 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp &&
920 (!mptcp_subflow_ctx(child) ||953 (!mptcp_subflow_ctx(child) ||
@@ -926,7 +959,8 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk,
926 tcp_rsk(req)->drop_req = true;959 tcp_rsk(req)->drop_req = true;
927 inet_csk_prepare_for_destroy_sock(child);960 inet_csk_prepare_for_destroy_sock(child);
928 tcp_done(child);961 tcp_done(child);
929- req->rsk_ops->send_reset(sk, skb);962+ reason = mptcp_get_rst_reason(skb);
963+ req->rsk_ops->send_reset(sk, skb, reason);
930 964 
931 /* The last child reference will be released by the caller */965 /* The last child reference will be released by the caller */
932 return child;966 return child;
@@ -947,7 +981,8 @@ enum mapping_status {
947 MAPPING_EMPTY,981 MAPPING_EMPTY,
948 MAPPING_DATA_FIN,982 MAPPING_DATA_FIN,
949 MAPPING_DUMMY,983 MAPPING_DUMMY,
950- MAPPING_BAD_CSUM984+ MAPPING_BAD_CSUM,
985+ MAPPING_NODSS
951};986};
952 987 
953static void dbg_bad_map(struct mptcp_subflow_context *subflow, u32 ssn)988static void dbg_bad_map(struct mptcp_subflow_context *subflow, u32 ssn)
@@ -1104,8 +1139,9 @@ static enum mapping_status get_mapping_status(struct sock *ssk,
1104 return MAPPING_EMPTY;1139 return MAPPING_EMPTY;
1105 }1140 }
1106 1141 
1142+ /* If the required DSS has likely been dropped by a middlebox */
1107 if (!subflow->map_valid)1143 if (!subflow->map_valid)
1108- return MAPPING_INVALID;1144+ return MAPPING_NODSS;
1109 1145 
1110 goto validate_seq;1146 goto validate_seq;
1111 }1147 }
@@ -1120,6 +1156,8 @@ static enum mapping_status get_mapping_status(struct sock *ssk,
1120 }1156 }
1121 1157 
1122 if (mpext->data_fin == 1) {1158 if (mpext->data_fin == 1) {
1159+ u64 data_fin_seq;
1160+ 
1123 if (data_len == 1) {1161 if (data_len == 1) {
1124 bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq,1162 bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq,
1125 mpext->dsn64);1163 mpext->dsn64);
@@ -1132,26 +1170,26 @@ static enum mapping_status get_mapping_status(struct sock *ssk,
1132 */1170 */
1133 skb_ext_del(skb, SKB_EXT_MPTCP);1171 skb_ext_del(skb, SKB_EXT_MPTCP);
1134 return MAPPING_OK;1172 return MAPPING_OK;
1135- } else {
1136- if (updated)
1137- mptcp_schedule_work((struct sock *)msk);
1138- 
1139- return MAPPING_DATA_FIN;
1140 }1173 }
1141- } else {
1142- u64 data_fin_seq = mpext->data_seq + data_len - 1;
1143 1174 
1144- /* If mpext->data_seq is a 32-bit value, data_fin_seq1175+ if (updated)
1145- * must also be limited to 32 bits.1176+ mptcp_schedule_work((struct sock *)msk);
1146- */
1147- if (!mpext->dsn64)
1148- data_fin_seq &= GENMASK_ULL(31, 0);
1149 1177 
1150- mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64);1178+ return MAPPING_DATA_FIN;
1151- pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d\n",
1152- data_fin_seq, mpext->dsn64);
1153 }1179 }
1154 1180 
1181+ data_fin_seq = mpext->data_seq + data_len - 1;
1182+ 
1183+ /* If mpext->data_seq is a 32-bit value, data_fin_seq must also
1184+ * be limited to 32 bits.
1185+ */
1186+ if (!mpext->dsn64)
1187+ data_fin_seq &= GENMASK_ULL(31, 0);
1188+ 
1189+ mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64);
1190+ pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d\n",
1191+ data_fin_seq, mpext->dsn64);
1192+ 
1155 /* Adjust for DATA_FIN using 1 byte of sequence space */1193 /* Adjust for DATA_FIN using 1 byte of sequence space */
1156 data_len--;1194 data_len--;
1157 }1195 }
@@ -1242,9 +1280,15 @@ static void mptcp_subflow_discard_data(struct sock *ssk, struct sk_buff *skb,
1242 subflow->map_valid = 0;1280 subflow->map_valid = 0;
1243}1281}
1244 1282 
1245-/* sched mptcp worker to remove the subflow if no more data is pending */1283+static bool subflow_is_done(const struct sock *sk)
1284+{
1285+ return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE;
1286+}
1287+ 
1288+/* sched mptcp worker for subflow cleanup if no more data is pending */
1246static void subflow_sched_work_if_closed(struct mptcp_sock *msk, struct sock *ssk)1289static void subflow_sched_work_if_closed(struct mptcp_sock *msk, struct sock *ssk)
1247{1290{
1291+ const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1248 struct sock *sk = (struct sock *)msk;1292 struct sock *sk = (struct sock *)msk;
1249 1293 
1250 if (likely(ssk->sk_state != TCP_CLOSE &&1294 if (likely(ssk->sk_state != TCP_CLOSE &&
@@ -1252,8 +1296,19 @@ static void subflow_sched_work_if_closed(struct mptcp_sock *msk, struct sock *ss
1252 inet_sk_state_load(sk) != TCP_ESTABLISHED)))1296 inet_sk_state_load(sk) != TCP_ESTABLISHED)))
1253 return;1297 return;
1254 1298 
1255- if (skb_queue_empty(&ssk->sk_receive_queue) &&1299+ if (!skb_queue_empty(&ssk->sk_receive_queue))
1256- !test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags))1300+ return;
1301+ 
1302+ if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags))
1303+ mptcp_schedule_work(sk);
1304+ 
1305+ /* when the fallback subflow closes the rx side, trigger a 'dummy'
1306+ * ingress data fin, so that the msk state will follow along
1307+ */
1308+ if (__mptcp_check_fallback(msk) && subflow_is_done(ssk) &&
1309+ msk->first == ssk &&
1310+ mptcp_update_rcv_data_fin(msk, subflow->map_seq +
1311+ subflow->map_data_len, true))
1257 mptcp_schedule_work(sk);1312 mptcp_schedule_work(sk);
1258}1313}
1259 1314 
@@ -1271,7 +1326,7 @@ static bool mptcp_subflow_fail(struct mptcp_sock *msk, struct sock *ssk)
1271 msk->allow_subflows = false;1326 msk->allow_subflows = false;
1272 spin_unlock_bh(&msk->fallback_lock);1327 spin_unlock_bh(&msk->fallback_lock);
1273 1328 
1274- /* greceful failure can happen only on the MPC subflow */1329+ /* graceful failure can happen only on the MPC subflow */
1275 if (WARN_ON_ONCE(ssk != READ_ONCE(msk->first)))1330 if (WARN_ON_ONCE(ssk != READ_ONCE(msk->first)))
1276 return false;1331 return false;
1277 1332 
@@ -1302,7 +1357,7 @@ static bool subflow_check_data_avail(struct sock *ssk)
1302 struct sk_buff *skb;1357 struct sk_buff *skb;
1303 1358 
1304 if (!skb_peek(&ssk->sk_receive_queue))1359 if (!skb_peek(&ssk->sk_receive_queue))
1305- WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_NODATA);1360+ WRITE_ONCE(subflow->data_avail, false);
1306 if (subflow->data_avail)1361 if (subflow->data_avail)
1307 return true;1362 return true;
1308 1363 
@@ -1314,7 +1369,7 @@ static bool subflow_check_data_avail(struct sock *ssk)
1314 status = get_mapping_status(ssk, msk);1369 status = get_mapping_status(ssk, msk);
1315 trace_subflow_check_data_avail(status, skb_peek(&ssk->sk_receive_queue));1370 trace_subflow_check_data_avail(status, skb_peek(&ssk->sk_receive_queue));
1316 if (unlikely(status == MAPPING_INVALID || status == MAPPING_DUMMY ||1371 if (unlikely(status == MAPPING_INVALID || status == MAPPING_DUMMY ||
1317- status == MAPPING_BAD_CSUM))1372+ status == MAPPING_BAD_CSUM || status == MAPPING_NODSS))
1318 goto fallback;1373 goto fallback;
1319 1374 
1320 if (status != MAPPING_OK)1375 if (status != MAPPING_OK)
@@ -1336,7 +1391,7 @@ static bool subflow_check_data_avail(struct sock *ssk)
1336 continue;1391 continue;
1337 }1392 }
1338 1393 
1339- WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_DATA_AVAIL);1394+ WRITE_ONCE(subflow->data_avail, true);
1340 break;1395 break;
1341 }1396 }
1342 return true;1397 return true;
@@ -1357,34 +1412,39 @@ static bool subflow_check_data_avail(struct sock *ssk)
1357 subflow->reset_reason = MPTCP_RST_EMIDDLEBOX;1412 subflow->reset_reason = MPTCP_RST_EMIDDLEBOX;
1358 goto reset;1413 goto reset;
1359 }1414 }
1360- WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_DATA_AVAIL);1415+ WRITE_ONCE(subflow->data_avail, true);
1361 return true;1416 return true;
1362 }1417 }
1363 1418 
1364- if (!mptcp_try_fallback(ssk)) {1419+ if (!mptcp_try_fallback(ssk, MPTCP_MIB_DSSFALLBACK)) {
1365 /* fatal protocol error, close the socket.1420 /* fatal protocol error, close the socket.
1366 * subflow_error_report() will introduce the appropriate barriers1421 * subflow_error_report() will introduce the appropriate barriers
1367 */1422 */
1368 subflow->reset_transient = 0;1423 subflow->reset_transient = 0;
1369- subflow->reset_reason = MPTCP_RST_EMPTCP;1424+ subflow->reset_reason = status == MAPPING_NODSS ?
1425+ MPTCP_RST_EMIDDLEBOX :
1426+ MPTCP_RST_EMPTCP;
1370 1427 
1371reset:1428reset:
1372 WRITE_ONCE(ssk->sk_err, EBADMSG);1429 WRITE_ONCE(ssk->sk_err, EBADMSG);
1373 tcp_set_state(ssk, TCP_CLOSE);1430 tcp_set_state(ssk, TCP_CLOSE);
1374 while ((skb = skb_peek(&ssk->sk_receive_queue)))1431 while ((skb = skb_peek(&ssk->sk_receive_queue)))
1375 sk_eat_skb(ssk, skb);1432 sk_eat_skb(ssk, skb);
1376- tcp_send_active_reset(ssk, GFP_ATOMIC);1433+ mptcp_send_active_reset_reason(ssk);
1377- WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_NODATA);1434+ WRITE_ONCE(subflow->data_avail, false);
1378 return false;1435 return false;
1379 }1436 }
1380 }1437 }
1381 1438 
1382 skb = skb_peek(&ssk->sk_receive_queue);1439 skb = skb_peek(&ssk->sk_receive_queue);
1383 subflow->map_valid = 1;1440 subflow->map_valid = 1;
1384- subflow->map_seq = READ_ONCE(msk->ack_seq);
1385 subflow->map_data_len = skb->len;1441 subflow->map_data_len = skb->len;
1386 subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq - subflow->ssn_offset;1442 subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq - subflow->ssn_offset;
1387- WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_DATA_AVAIL);1443+ subflow->map_seq = __mptcp_expand_seq(subflow->map_seq,
1444+ subflow->iasn +
1445+ TCP_SKB_CB(skb)->seq -
1446+ subflow->ssn_offset - 1);
1447+ WRITE_ONCE(subflow->data_avail, true);
1388 return true;1448 return true;
1389}1449}
1390 1450 
@@ -1396,7 +1456,7 @@ bool mptcp_subflow_data_available(struct sock *sk)
1396 if (subflow->map_valid &&1456 if (subflow->map_valid &&
1397 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) {1457 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) {
1398 subflow->map_valid = 0;1458 subflow->map_valid = 0;
1399- WRITE_ONCE(subflow->data_avail, MPTCP_SUBFLOW_NODATA);1459+ WRITE_ONCE(subflow->data_avail, false);
1400 1460 
1401 pr_debug("Done with mapping: seq=%u data_len=%u\n",1461 pr_debug("Done with mapping: seq=%u data_len=%u\n",
1402 subflow->map_subflow_seq,1462 subflow->map_subflow_seq,
@@ -1550,28 +1610,31 @@ void mptcp_info2sockaddr(const struct mptcp_addr_info *info,
1550#endif1610#endif
1551}1611}
1552 1612 
1553-int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc,1613+int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_pm_local *local,
1554 const struct mptcp_addr_info *remote)1614 const struct mptcp_addr_info *remote)
1555{1615{
1556 struct mptcp_sock *msk = mptcp_sk(sk);1616 struct mptcp_sock *msk = mptcp_sk(sk);
1557 struct mptcp_subflow_context *subflow;1617 struct mptcp_subflow_context *subflow;
1618+ int local_id = local->addr.id;
1558 struct sockaddr_storage addr;1619 struct sockaddr_storage addr;
1559 int remote_id = remote->id;1620 int remote_id = remote->id;
1560- int local_id = loc->id;
1561 int err = -ENOTCONN;1621 int err = -ENOTCONN;
1562 struct socket *sf;1622 struct socket *sf;
1563 struct sock *ssk;1623 struct sock *ssk;
1564 u32 remote_token;1624 u32 remote_token;
1565 int addrlen;1625 int addrlen;
1566- int ifindex;
1567- u8 flags;
1568 1626 
1627+ /* The userspace PM sent the request too early? */
1569 if (!mptcp_is_fully_established(sk))1628 if (!mptcp_is_fully_established(sk))
1570 goto err_out;1629 goto err_out;
1571 1630 
1572- err = mptcp_subflow_create_socket(sk, loc->family, &sf);1631+ err = mptcp_subflow_create_socket(sk, local->addr.family, &sf);
1573- if (err)1632+ if (err) {
1633+ MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTXCREATSKERR);
1634+ pr_debug("msk=%p local=%d remote=%d create sock error: %d\n",
1635+ msk, local_id, remote_id, err);
1574 goto err_out;1636 goto err_out;
1637+ }
1575 1638 
1576 ssk = sf->sk;1639 ssk = sf->sk;
1577 subflow = mptcp_subflow_ctx(ssk);1640 subflow = mptcp_subflow_ctx(ssk);
@@ -1579,28 +1642,39 @@ int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc,
1579 get_random_bytes(&subflow->local_nonce, sizeof(u32));1642 get_random_bytes(&subflow->local_nonce, sizeof(u32));
1580 } while (!subflow->local_nonce);1643 } while (!subflow->local_nonce);
1581 1644 
1582- if (local_id)1645+ /* if 'IPADDRANY', the ID will be set later, after the routing */
1646+ if (local->addr.family == AF_INET) {
1647+ if (!local->addr.addr.s_addr)
1648+ local_id = -1;
1649+#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1650+ } else if (sk->sk_family == AF_INET6) {
1651+ if (ipv6_addr_any(&local->addr.addr6))
1652+ local_id = -1;
1653+#endif
1654+ }
1655+ 
1656+ if (local_id >= 0)
1583 subflow_set_local_id(subflow, local_id);1657 subflow_set_local_id(subflow, local_id);
1584 1658 
1585- mptcp_pm_get_flags_and_ifindex_by_id(msk, local_id,
1586- &flags, &ifindex);
1587 subflow->remote_key_valid = 1;1659 subflow->remote_key_valid = 1;
1588- subflow->remote_key = msk->remote_key;1660+ subflow->remote_key = READ_ONCE(msk->remote_key);
1589- subflow->local_key = msk->local_key;1661+ subflow->local_key = READ_ONCE(msk->local_key);
1590 subflow->token = msk->token;1662 subflow->token = msk->token;
1591- mptcp_info2sockaddr(loc, &addr, ssk->sk_family);1663+ mptcp_info2sockaddr(&local->addr, &addr, ssk->sk_family);
1592 1664 
1593 addrlen = sizeof(struct sockaddr_in);1665 addrlen = sizeof(struct sockaddr_in);
1594#if IS_ENABLED(CONFIG_MPTCP_IPV6)1666#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1595 if (addr.ss_family == AF_INET6)1667 if (addr.ss_family == AF_INET6)
1596 addrlen = sizeof(struct sockaddr_in6);1668 addrlen = sizeof(struct sockaddr_in6);
1597#endif1669#endif
1598- mptcp_sockopt_sync(msk, ssk);1670+ ssk->sk_bound_dev_if = local->ifindex;
1599- 1671+ err = kernel_bind(sf, (struct sockaddr_unsized *)&addr, addrlen);
1600- ssk->sk_bound_dev_if = ifindex;1672+ if (err) {
1601- err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen);1673+ MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTXBINDERR);
1602- if (err)1674+ pr_debug("msk=%p local=%d remote=%d bind error: %d\n",
1675+ msk, local_id, remote_id, err);
1603 goto failed;1676 goto failed;
1677+ }
1604 1678 
1605 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL);1679 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL);
1606 pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d\n", msk,1680 pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d\n", msk,
@@ -1608,15 +1682,21 @@ int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc,
1608 subflow->remote_token = remote_token;1682 subflow->remote_token = remote_token;
1609 WRITE_ONCE(subflow->remote_id, remote_id);1683 WRITE_ONCE(subflow->remote_id, remote_id);
1610 subflow->request_join = 1;1684 subflow->request_join = 1;
1611- subflow->request_bkup = !!(flags & MPTCP_PM_ADDR_FLAG_BACKUP);1685+ subflow->request_bkup = !!(local->flags & MPTCP_PM_ADDR_FLAG_BACKUP);
1612 subflow->subflow_id = msk->subflow_id++;1686 subflow->subflow_id = msk->subflow_id++;
1613 mptcp_info2sockaddr(remote, &addr, ssk->sk_family);1687 mptcp_info2sockaddr(remote, &addr, ssk->sk_family);
1614 1688 
1615 sock_hold(ssk);1689 sock_hold(ssk);
1616 list_add_tail(&subflow->node, &msk->conn_list);1690 list_add_tail(&subflow->node, &msk->conn_list);
1617- err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK);1691+ err = kernel_connect(sf, (struct sockaddr_unsized *)&addr, addrlen, O_NONBLOCK);
1618- if (err && err != -EINPROGRESS)1692+ if (err && err != -EINPROGRESS) {
1693+ MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTXCONNECTERR);
1694+ pr_debug("msk=%p local=%d remote=%d connect error: %d\n",
1695+ msk, local_id, remote_id, err);
1619 goto failed_unlink;1696 goto failed_unlink;
1697+ }
1698+ 
1699+ MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTX);
1620 1700 
1621 /* discard the subflow socket */1701 /* discard the subflow socket */
1622 mptcp_sock_graft(ssk, sk->sk_socket);1702 mptcp_sock_graft(ssk, sk->sk_socket);
@@ -1640,30 +1720,39 @@ int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc,
1640 return err;1720 return err;
1641}1721}
1642 1722 
1643-static void mptcp_attach_cgroup(struct sock *parent, struct sock *child)1723+void __mptcp_inherit_memcg(struct sock *sk, struct sock *ssk, gfp_t gfp)
1724+{
1725+ /* Only if the msk has been accepted already (and not orphaned).*/
1726+ if (!mem_cgroup_sockets_enabled || !sk->sk_socket)
1727+ return;
1728+ 
1729+ mem_cgroup_sk_inherit(sk, ssk);
1730+ __sk_charge(ssk, gfp);
1731+}
1732+ 
1733+void __mptcp_inherit_cgrp_data(struct sock *sk, struct sock *ssk)
1644{1734{
1645#ifdef CONFIG_SOCK_CGROUP_DATA1735#ifdef CONFIG_SOCK_CGROUP_DATA
1646- struct sock_cgroup_data *parent_skcd = &parent->sk_cgrp_data,1736+ struct sock_cgroup_data *sk_cd = &sk->sk_cgrp_data,
1647- *child_skcd = &child->sk_cgrp_data;1737+ *ssk_cd = &ssk->sk_cgrp_data;
1648 1738 
1649 /* only the additional subflows created by kworkers have to be modified */1739 /* only the additional subflows created by kworkers have to be modified */
1650- if (cgroup_id(sock_cgroup_ptr(parent_skcd)) !=1740+ if (cgroup_id(sock_cgroup_ptr(sk_cd)) !=
1651- cgroup_id(sock_cgroup_ptr(child_skcd))) {1741+ cgroup_id(sock_cgroup_ptr(ssk_cd))) {
1652-#ifdef CONFIG_MEMCG1742+ cgroup_sk_free(ssk_cd);
1653- struct mem_cgroup *memcg = parent->sk_memcg;1743+ *ssk_cd = *sk_cd;
1654- 1744+ cgroup_sk_clone(sk_cd);
1655- mem_cgroup_sk_free(child);
1656- if (memcg && css_tryget(&memcg->css))
1657- child->sk_memcg = memcg;
1658-#endif /* CONFIG_MEMCG */
1659- 
1660- cgroup_sk_free(child_skcd);
1661- *child_skcd = *parent_skcd;
1662- cgroup_sk_clone(child_skcd);
1663 }1745 }
1664#endif /* CONFIG_SOCK_CGROUP_DATA */1746#endif /* CONFIG_SOCK_CGROUP_DATA */
1665}1747}
1666 1748 
1749+static void mptcp_attach_cgroup(struct sock *parent, struct sock *child)
1750+{
1751+ __mptcp_inherit_cgrp_data(parent, child);
1752+ if (mem_cgroup_sockets_enabled)
1753+ mem_cgroup_sk_inherit(parent, child);
1754+}
1755+ 
1667static void mptcp_subflow_ops_override(struct sock *ssk)1756static void mptcp_subflow_ops_override(struct sock *ssk)
1668{1757{
1669#if IS_ENABLED(CONFIG_MPTCP_IPV6)1758#if IS_ENABLED(CONFIG_MPTCP_IPV6)
@@ -1706,7 +1795,7 @@ int mptcp_subflow_create_socket(struct sock *sk, unsigned short family,
1706 1795 
1707 err = security_mptcp_add_subflow(sk, sf->sk);1796 err = security_mptcp_add_subflow(sk, sf->sk);
1708 if (err)1797 if (err)
1709- goto release_ssk;1798+ goto err_free;
1710 1799 
1711 /* the newly created socket has to be in the same cgroup as its parent */1800 /* the newly created socket has to be in the same cgroup as its parent */
1712 mptcp_attach_cgroup(sk, sf->sk);1801 mptcp_attach_cgroup(sk, sf->sk);
@@ -1715,21 +1804,15 @@ int mptcp_subflow_create_socket(struct sock *sk, unsigned short family,
1715 * needs it.1804 * needs it.
1716 * Update ns_tracker to current stack trace and refcounted tracker.1805 * Update ns_tracker to current stack trace and refcounted tracker.
1717 */1806 */
1718- __netns_tracker_free(net, &sf->sk->ns_tracker, false);1807+ sk_net_refcnt_upgrade(sf->sk);
1719- sf->sk->sk_net_refcnt = 1;
1720- get_net_track(net, &sf->sk->ns_tracker, GFP_KERNEL);
1721- sock_inuse_add(net, 1);
1722 err = tcp_set_ulp(sf->sk, "mptcp");1808 err = tcp_set_ulp(sf->sk, "mptcp");
1809+ if (err)
1810+ goto err_free;
1723 1811 
1724-release_ssk:1812+ mptcp_sockopt_sync_locked(mptcp_sk(sk), sf->sk);
1725 release_sock(sf->sk);1813 release_sock(sf->sk);
1726 1814 
1727- if (err) {1815+ /* the newly created socket really belongs to the owning MPTCP
1728- sock_release(sf);
1729- return err;
1730- }
1731- 
1732- /* the newly created socket really belongs to the owning MPTCP master
1733 * socket, even if for additional subflows the allocation is performed1816 * socket, even if for additional subflows the allocation is performed
1734 * by a kernel workqueue. Adjust inode references, so that the1817 * by a kernel workqueue. Adjust inode references, so that the
1735 * procfs/diag interfaces really show this one belonging to the correct1818 * procfs/diag interfaces really show this one belonging to the correct
@@ -1748,6 +1831,11 @@ int mptcp_subflow_create_socket(struct sock *sk, unsigned short family,
1748 mptcp_subflow_ops_override(sf->sk);1831 mptcp_subflow_ops_override(sf->sk);
1749 1832 
1750 return 0;1833 return 0;
1834+ 
1835+err_free:
1836+ release_sock(sf->sk);
1837+ sock_release(sf);
1838+ return err;
1751}1839}
1752 1840 
1753static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk,1841static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk,
@@ -1783,27 +1871,13 @@ static void __subflow_state_change(struct sock *sk)
1783 rcu_read_unlock();1871 rcu_read_unlock();
1784}1872}
1785 1873 
1786-static bool subflow_is_done(const struct sock *sk)
1787-{
1788- return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE;
1789-}
1790- 
1791static void subflow_state_change(struct sock *sk)1874static void subflow_state_change(struct sock *sk)
1792{1875{
1793 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);1876 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1794 struct sock *parent = subflow->conn;1877 struct sock *parent = subflow->conn;
1795- struct mptcp_sock *msk;
1796 1878 
1797 __subflow_state_change(sk);1879 __subflow_state_change(sk);
1798 1880 
1799- msk = mptcp_sk(parent);
1800- if (subflow_simultaneous_connect(sk)) {
1801- WARN_ON_ONCE(!mptcp_try_fallback(sk));
1802- pr_fallback(msk);
1803- subflow->conn_finished = 1;
1804- mptcp_propagate_state(parent, sk, subflow, NULL);
1805- }
1806- 
1807 /* as recvmsg() does not acquire the subflow socket for ssk selection1881 /* as recvmsg() does not acquire the subflow socket for ssk selection
1808 * a fin packet carrying a DSS can be unnoticed if we don't trigger1882 * a fin packet carrying a DSS can be unnoticed if we don't trigger
1809 * the data available machinery here.1883 * the data available machinery here.
@@ -1814,13 +1888,6 @@ static void subflow_state_change(struct sock *sk)
1814 subflow_error_report(sk);1888 subflow_error_report(sk);
1815 1889 
1816 subflow_sched_work_if_closed(mptcp_sk(parent), sk);1890 subflow_sched_work_if_closed(mptcp_sk(parent), sk);
1817- 
1818- /* when the fallback subflow closes the rx side, trigger a 'dummy'
1819- * ingress data fin, so that the msk state will follow along
1820- */
1821- if (__mptcp_check_fallback(msk) && subflow_is_done(sk) && msk->first == sk &&
1822- mptcp_update_rcv_data_fin(msk, READ_ONCE(msk->ack_seq), true))
1823- mptcp_schedule_work(parent);
1824}1891}
1825 1892 
1826void mptcp_subflow_queue_clean(struct sock *listener_sk, struct sock *listener_ssk)1893void mptcp_subflow_queue_clean(struct sock *listener_sk, struct sock *listener_ssk)
@@ -1990,7 +2057,6 @@ static void subflow_ulp_clone(const struct request_sock *req,
1990 new_ctx->tcp_state_change = old_ctx->tcp_state_change;2057 new_ctx->tcp_state_change = old_ctx->tcp_state_change;
1991 new_ctx->tcp_error_report = old_ctx->tcp_error_report;2058 new_ctx->tcp_error_report = old_ctx->tcp_error_report;
1992 new_ctx->rel_write_seq = 1;2059 new_ctx->rel_write_seq = 1;
1993- new_ctx->tcp_sock = newsk;
1994 2060 
1995 if (subflow_req->mp_capable) {2061 if (subflow_req->mp_capable) {
1996 /* see comments in subflow_syn_recv_sock(), MPTCP connection2062 /* see comments in subflow_syn_recv_sock(), MPTCP connection
@@ -2007,7 +2073,7 @@ static void subflow_ulp_clone(const struct request_sock *req,
2007 } else if (subflow_req->mp_join) {2073 } else if (subflow_req->mp_join) {
2008 new_ctx->ssn_offset = subflow_req->ssn_offset;2074 new_ctx->ssn_offset = subflow_req->ssn_offset;
2009 new_ctx->mp_join = 1;2075 new_ctx->mp_join = 1;
2010- new_ctx->fully_established = 1;2076+ WRITE_ONCE(new_ctx->fully_established, true);
2011 new_ctx->remote_key_valid = 1;2077 new_ctx->remote_key_valid = 1;
2012 new_ctx->backup = subflow_req->backup;2078 new_ctx->backup = subflow_req->backup;
2013 new_ctx->request_bkup = subflow_req->request_bkup;2079 new_ctx->request_bkup = subflow_req->request_bkup;
@@ -2094,6 +2160,10 @@ void __init mptcp_subflow_init(void)
2094 tcp_prot_override = tcp_prot;2160 tcp_prot_override = tcp_prot;
2095 tcp_prot_override.release_cb = tcp_release_cb_override;2161 tcp_prot_override.release_cb = tcp_release_cb_override;
2096 tcp_prot_override.diag_destroy = tcp_abort_override;2162 tcp_prot_override.diag_destroy = tcp_abort_override;
2163+#ifdef CONFIG_BPF_SYSCALL
2164+ /* Disable sockmap processing for subflows */
2165+ tcp_prot_override.psock_update_sk_prot = NULL;
2166+#endif
2097 2167 
2098#if IS_ENABLED(CONFIG_MPTCP_IPV6)2168#if IS_ENABLED(CONFIG_MPTCP_IPV6)
2099 /* In struct mptcp_subflow_request_sock, we assume the TCP request sock2169 /* In struct mptcp_subflow_request_sock, we assume the TCP request sock
@@ -2125,12 +2195,15 @@ void __init mptcp_subflow_init(void)
2125 subflow_v6m_specific.send_check = ipv4_specific.send_check;2195 subflow_v6m_specific.send_check = ipv4_specific.send_check;
2126 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len;2196 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len;
2127 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced;2197 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced;
2128- subflow_v6m_specific.net_frag_header_len = 0;
2129 subflow_v6m_specific.rebuild_header = subflow_rebuild_header;2198 subflow_v6m_specific.rebuild_header = subflow_rebuild_header;
2130 2199 
2131 tcpv6_prot_override = tcpv6_prot;2200 tcpv6_prot_override = tcpv6_prot;
2132 tcpv6_prot_override.release_cb = tcp_release_cb_override;2201 tcpv6_prot_override.release_cb = tcp_release_cb_override;
2133 tcpv6_prot_override.diag_destroy = tcp_abort_override;2202 tcpv6_prot_override.diag_destroy = tcp_abort_override;
2203+#ifdef CONFIG_BPF_SYSCALL
2204+ /* Disable sockmap processing for subflows */
2205+ tcpv6_prot_override.psock_update_sk_prot = NULL;
2206+#endif
2134#endif2207#endif
2135 2208 
2136 mptcp_diag_subflow_init(&subflow_ulp_ops);2209 mptcp_diag_subflow_init(&subflow_ulp_ops);
Mnet/sched/sch_cake.c+145-123
@@ -361,8 +361,24 @@ static const u8 besteffort[] = {
361static const u8 normal_order[] = {0, 1, 2, 3, 4, 5, 6, 7};361static const u8 normal_order[] = {0, 1, 2, 3, 4, 5, 6, 7};
362static const u8 bulk_order[] = {1, 0, 2, 3};362static const u8 bulk_order[] = {1, 0, 2, 3};
363 363 
364+/* There is a big difference in timing between the accurate values placed in the
365+ * cache and the approximations given by a single Newton step for small count
366+ * values, particularly when stepping from count 1 to 2 or vice versa. Hence,
367+ * these values are calculated using eight Newton steps, using the
368+ * implementation below. Above 16, a single Newton step gives sufficient
369+ * accuracy in either direction, given the precision stored.
370+ *
371+ * The magnitude of the error when stepping up to count 2 is such as to give the
372+ * value that *should* have been produced at count 4.
373+ */
374+ 
364#define REC_INV_SQRT_CACHE (16)375#define REC_INV_SQRT_CACHE (16)
365-static u32 cobalt_rec_inv_sqrt_cache[REC_INV_SQRT_CACHE] = {0};376+static const u32 inv_sqrt_cache[REC_INV_SQRT_CACHE] = {
377+ ~0, ~0, 3037000500, 2479700525,
378+ 2147483647, 1920767767, 1753413056, 1623345051,
379+ 1518500250, 1431655765, 1358187914, 1294981364,
380+ 1239850263, 1191209601, 1147878294, 1108955788
381+};
366 382 
367/* http://en.wikipedia.org/wiki/Methods_of_computing_square_roots383/* http://en.wikipedia.org/wiki/Methods_of_computing_square_roots
368 * new_invsqrt = (invsqrt / 2) * (3 - count * invsqrt^2)384 * new_invsqrt = (invsqrt / 2) * (3 - count * invsqrt^2)
@@ -388,47 +404,14 @@ static void cobalt_newton_step(struct cobalt_vars *vars)
388static void cobalt_invsqrt(struct cobalt_vars *vars)404static void cobalt_invsqrt(struct cobalt_vars *vars)
389{405{
390 if (vars->count < REC_INV_SQRT_CACHE)406 if (vars->count < REC_INV_SQRT_CACHE)
391- vars->rec_inv_sqrt = cobalt_rec_inv_sqrt_cache[vars->count];407+ vars->rec_inv_sqrt = inv_sqrt_cache[vars->count];
392 else408 else
393 cobalt_newton_step(vars);409 cobalt_newton_step(vars);
394}410}
395 411 
396-/* There is a big difference in timing between the accurate values placed in
397- * the cache and the approximations given by a single Newton step for small
398- * count values, particularly when stepping from count 1 to 2 or vice versa.
399- * Above 16, a single Newton step gives sufficient accuracy in either
400- * direction, given the precision stored.
401- *
402- * The magnitude of the error when stepping up to count 2 is such as to give
403- * the value that *should* have been produced at count 4.
404- */
405- 
406-static void cobalt_cache_init(void)
407-{
408- struct cobalt_vars v;
409- 
410- memset(&v, 0, sizeof(v));
411- v.rec_inv_sqrt = ~0U;
412- cobalt_rec_inv_sqrt_cache[0] = v.rec_inv_sqrt;
413- 
414- for (v.count = 1; v.count < REC_INV_SQRT_CACHE; v.count++) {
415- cobalt_newton_step(&v);
416- cobalt_newton_step(&v);
417- cobalt_newton_step(&v);
418- cobalt_newton_step(&v);
419- 
420- cobalt_rec_inv_sqrt_cache[v.count] = v.rec_inv_sqrt;
421- }
422-}
423- 
424static void cobalt_vars_init(struct cobalt_vars *vars)412static void cobalt_vars_init(struct cobalt_vars *vars)
425{413{
426 memset(vars, 0, sizeof(*vars));414 memset(vars, 0, sizeof(*vars));
427- 
428- if (!cobalt_rec_inv_sqrt_cache[0]) {
429- cobalt_cache_init();
430- cobalt_rec_inv_sqrt_cache[0] = ~0;
431- }
432}415}
433 416 
434/* CoDel control_law is t + interval/sqrt(count)417/* CoDel control_law is t + interval/sqrt(count)
@@ -501,13 +484,14 @@ static bool cobalt_queue_empty(struct cobalt_vars *vars,
501/* Call this with a freshly dequeued packet for possible congestion marking.484/* Call this with a freshly dequeued packet for possible congestion marking.
502 * Returns true as an instruction to drop the packet, false for delivery.485 * Returns true as an instruction to drop the packet, false for delivery.
503 */486 */
504-static bool cobalt_should_drop(struct cobalt_vars *vars,487+static enum skb_drop_reason cobalt_should_drop(struct cobalt_vars *vars,
505- struct cobalt_params *p,488+ struct cobalt_params *p,
506- ktime_t now,489+ ktime_t now,
507- struct sk_buff *skb,490+ struct sk_buff *skb,
508- u32 bulk_flows)491+ u32 bulk_flows)
509{492{
510- bool next_due, over_target, drop = false;493+ enum skb_drop_reason reason = SKB_NOT_DROPPED_YET;
494+ bool next_due, over_target;
511 ktime_t schedule;495 ktime_t schedule;
512 u64 sojourn;496 u64 sojourn;
513 497 
@@ -550,7 +534,8 @@ static bool cobalt_should_drop(struct cobalt_vars *vars,
550 534 
551 if (next_due && vars->dropping) {535 if (next_due && vars->dropping) {
552 /* Use ECN mark if possible, otherwise drop */536 /* Use ECN mark if possible, otherwise drop */
553- drop = !(vars->ecn_marked = INET_ECN_set_ce(skb));537+ if (!(vars->ecn_marked = INET_ECN_set_ce(skb)))
538+ reason = SKB_DROP_REASON_QDISC_CONGESTED;
554 539 
555 vars->count++;540 vars->count++;
556 if (!vars->count)541 if (!vars->count)
@@ -573,16 +558,17 @@ static bool cobalt_should_drop(struct cobalt_vars *vars,
573 }558 }
574 559 
575 /* Simple BLUE implementation. Lack of ECN is deliberate. */560 /* Simple BLUE implementation. Lack of ECN is deliberate. */
576- if (vars->p_drop)561+ if (vars->p_drop && reason == SKB_NOT_DROPPED_YET &&
577- drop |= (get_random_u32() < vars->p_drop);562+ get_random_u32() < vars->p_drop)
563+ reason = SKB_DROP_REASON_CAKE_FLOOD;
578 564 
579 /* Overload the drop_next field as an activity timeout */565 /* Overload the drop_next field as an activity timeout */
580 if (!vars->count)566 if (!vars->count)
581 vars->drop_next = ktime_add_ns(now, p->interval);567 vars->drop_next = ktime_add_ns(now, p->interval);
582- else if (ktime_to_ns(schedule) > 0 && !drop)568+ else if (ktime_to_ns(schedule) > 0 && reason == SKB_NOT_DROPPED_YET)
583 vars->drop_next = now;569 vars->drop_next = now;
584 570 
585- return drop;571+ return reason;
586}572}
587 573 
588static bool cake_update_flowkeys(struct flow_keys *keys,574static bool cake_update_flowkeys(struct flow_keys *keys,
@@ -1421,7 +1407,10 @@ static u32 cake_overhead(struct cake_sched_data *q, const struct sk_buff *skb)
1421 return cake_calc_overhead(q, len, off);1407 return cake_calc_overhead(q, len, off);
1422 1408 
1423 /* borrowed from qdisc_pkt_len_init() */1409 /* borrowed from qdisc_pkt_len_init() */
1424- hdr_len = skb_transport_offset(skb);1410+ if (!skb->encapsulation)
1411+ hdr_len = skb_transport_offset(skb);
1412+ else
1413+ hdr_len = skb_inner_transport_offset(skb);
1425 1414 
1426 /* + transport layer */1415 /* + transport layer */
1427 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 |1416 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 |
@@ -1572,7 +1561,7 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free)
1572 if (!q->overflow_timeout) {1561 if (!q->overflow_timeout) {
1573 int i;1562 int i;
1574 /* Build fresh max-heap */1563 /* Build fresh max-heap */
1575- for (i = CAKE_MAX_TINS * CAKE_QUEUES / 2; i >= 0; i--)1564+ for (i = CAKE_MAX_TINS * CAKE_QUEUES / 2 - 1; i >= 0; i--)
1576 cake_heapify(q, i);1565 cake_heapify(q, i);
1577 }1566 }
1578 q->overflow_timeout = 65535;1567 q->overflow_timeout = 65535;
@@ -1602,14 +1591,12 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free)
1602 1591 
1603 flow->dropped++;1592 flow->dropped++;
1604 b->tin_dropped++;1593 b->tin_dropped++;
1605- sch->qstats.drops++;
1606 1594 
1607 if (q->rate_flags & CAKE_FLAG_INGRESS)1595 if (q->rate_flags & CAKE_FLAG_INGRESS)
1608 cake_advance_shaper(q, b, skb, now, true);1596 cake_advance_shaper(q, b, skb, now, true);
1609 1597 
1610- __qdisc_drop(skb, to_free);1598+ qdisc_drop_reason(skb, sch, to_free, SKB_DROP_REASON_QDISC_OVERLIMIT);
1611 sch->q.qlen--;1599 sch->q.qlen--;
1612- qdisc_tree_reduce_backlog(sch, 1, len);
1613 1600 
1614 cake_heapify(q, 0);1601 cake_heapify(q, 0);
1615 1602 
@@ -1755,14 +1742,14 @@ static void cake_reconfigure(struct Qdisc *sch);
1755static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,1742static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
1756 struct sk_buff **to_free)1743 struct sk_buff **to_free)
1757{1744{
1745+ u32 idx, tin, prev_qlen, prev_backlog, drop_id;
1758 struct cake_sched_data *q = qdisc_priv(sch);1746 struct cake_sched_data *q = qdisc_priv(sch);
1759- int len = qdisc_pkt_len(skb);1747+ int len = qdisc_pkt_len(skb), ret;
1760- int ret;
1761 struct sk_buff *ack = NULL;1748 struct sk_buff *ack = NULL;
1762 ktime_t now = ktime_get();1749 ktime_t now = ktime_get();
1763 struct cake_tin_data *b;1750 struct cake_tin_data *b;
1764 struct cake_flow *flow;1751 struct cake_flow *flow;
1765- u32 idx;1752+ bool same_flow = false;
1766 1753 
1767 /* choose flow to insert into */1754 /* choose flow to insert into */
1768 idx = cake_classify(sch, &b, skb, q->flow_mode, &ret);1755 idx = cake_classify(sch, &b, skb, q->flow_mode, &ret);
@@ -1772,6 +1759,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
1772 __qdisc_drop(skb, to_free);1759 __qdisc_drop(skb, to_free);
1773 return ret;1760 return ret;
1774 }1761 }
1762+ tin = (u32)(b - q->tins);
1775 idx--;1763 idx--;
1776 flow = &b->flows[idx];1764 flow = &b->flows[idx];
1777 1765 
@@ -1834,6 +1822,8 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
1834 consume_skb(skb);1822 consume_skb(skb);
1835 } else {1823 } else {
1836 /* not splitting */1824 /* not splitting */
1825+ int ack_pkt_len = 0;
1826+ 
1837 cobalt_set_enqueue_time(skb, now);1827 cobalt_set_enqueue_time(skb, now);
1838 get_cobalt_cb(skb)->adjusted_len = cake_overhead(q, skb);1828 get_cobalt_cb(skb)->adjusted_len = cake_overhead(q, skb);
1839 flow_queue_add(flow, skb);1829 flow_queue_add(flow, skb);
@@ -1844,13 +1834,13 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
1844 if (ack) {1834 if (ack) {
1845 b->ack_drops++;1835 b->ack_drops++;
1846 sch->qstats.drops++;1836 sch->qstats.drops++;
1847- b->bytes += qdisc_pkt_len(ack);1837+ ack_pkt_len = qdisc_pkt_len(ack);
1848- len -= qdisc_pkt_len(ack);1838+ b->bytes += ack_pkt_len;
1849 q->buffer_used += skb->truesize - ack->truesize;1839 q->buffer_used += skb->truesize - ack->truesize;
1850 if (q->rate_flags & CAKE_FLAG_INGRESS)1840 if (q->rate_flags & CAKE_FLAG_INGRESS)
1851 cake_advance_shaper(q, b, ack, now, true);1841 cake_advance_shaper(q, b, ack, now, true);
1852 1842 
1853- qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(ack));1843+ qdisc_tree_reduce_backlog(sch, 1, ack_pkt_len);
1854 consume_skb(ack);1844 consume_skb(ack);
1855 } else {1845 } else {
1856 sch->q.qlen++;1846 sch->q.qlen++;
@@ -1859,11 +1849,11 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
1859 1849 
1860 /* stats */1850 /* stats */
1861 b->packets++;1851 b->packets++;
1862- b->bytes += len;1852+ b->bytes += len - ack_pkt_len;
1863- b->backlogs[idx] += len;1853+ b->backlogs[idx] += len - ack_pkt_len;
1864- b->tin_backlog += len;1854+ b->tin_backlog += len - ack_pkt_len;
1865- sch->qstats.backlog += len;1855+ sch->qstats.backlog += len - ack_pkt_len;
1866- q->avg_window_bytes += len;1856+ q->avg_window_bytes += len - ack_pkt_len;
1867 }1857 }
1868 1858 
1869 if (q->overflow_timeout)1859 if (q->overflow_timeout)
@@ -1938,15 +1928,29 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
1938 if (q->buffer_used > q->buffer_max_used)1928 if (q->buffer_used > q->buffer_max_used)
1939 q->buffer_max_used = q->buffer_used;1929 q->buffer_max_used = q->buffer_used;
1940 1930 
1941- if (q->buffer_used > q->buffer_limit) {1931+ if (q->buffer_used <= q->buffer_limit)
1942- u32 dropped = 0;1932+ return NET_XMIT_SUCCESS;
1943 1933 
1944- while (q->buffer_used > q->buffer_limit) {1934+ prev_qlen = sch->q.qlen;
1945- dropped++;1935+ prev_backlog = sch->qstats.backlog;
1946- cake_drop(sch, to_free);1936+ 
1947- }1937+ while (q->buffer_used > q->buffer_limit) {
1948- b->drop_overlimit += dropped;1938+ drop_id = cake_drop(sch, to_free);
1939+ if ((drop_id >> 16) == tin &&
1940+ (drop_id & 0xFFFF) == idx)
1941+ same_flow = true;
1949 }1942 }
1943+ 
1944+ prev_qlen -= sch->q.qlen;
1945+ prev_backlog -= sch->qstats.backlog;
1946+ b->drop_overlimit += prev_qlen;
1947+ 
1948+ if (same_flow) {
1949+ qdisc_tree_reduce_backlog(sch, prev_qlen - 1,
1950+ prev_backlog - len);
1951+ return NET_XMIT_CN;
1952+ }
1953+ qdisc_tree_reduce_backlog(sch, prev_qlen, prev_backlog);
1950 return NET_XMIT_SUCCESS;1954 return NET_XMIT_SUCCESS;
1951}1955}
1952 1956 
@@ -1982,13 +1986,14 @@ static void cake_clear_tin(struct Qdisc *sch, u16 tin)
1982 q->cur_tin = tin;1986 q->cur_tin = tin;
1983 for (q->cur_flow = 0; q->cur_flow < CAKE_QUEUES; q->cur_flow++)1987 for (q->cur_flow = 0; q->cur_flow < CAKE_QUEUES; q->cur_flow++)
1984 while (!!(skb = cake_dequeue_one(sch)))1988 while (!!(skb = cake_dequeue_one(sch)))
1985- kfree_skb(skb);1989+ kfree_skb_reason(skb, SKB_DROP_REASON_QUEUE_PURGE);
1986}1990}
1987 1991 
1988static struct sk_buff *cake_dequeue(struct Qdisc *sch)1992static struct sk_buff *cake_dequeue(struct Qdisc *sch)
1989{1993{
1990 struct cake_sched_data *q = qdisc_priv(sch);1994 struct cake_sched_data *q = qdisc_priv(sch);
1991 struct cake_tin_data *b = &q->tins[q->cur_tin];1995 struct cake_tin_data *b = &q->tins[q->cur_tin];
1996+ enum skb_drop_reason reason;
1992 ktime_t now = ktime_get();1997 ktime_t now = ktime_get();
1993 struct cake_flow *flow;1998 struct cake_flow *flow;
1994 struct list_head *head;1999 struct list_head *head;
@@ -2170,12 +2175,12 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch)
2170 goto begin;2175 goto begin;
2171 }2176 }
2172 2177 
2178+ reason = cobalt_should_drop(&flow->cvars, &b->cparams, now, skb,
2179+ (b->bulk_flow_count *
2180+ !!(q->rate_flags &
2181+ CAKE_FLAG_INGRESS)));
2173 /* Last packet in queue may be marked, shouldn't be dropped */2182 /* Last packet in queue may be marked, shouldn't be dropped */
2174- if (!cobalt_should_drop(&flow->cvars, &b->cparams, now, skb,2183+ if (reason == SKB_NOT_DROPPED_YET || !flow->head)
2175- (b->bulk_flow_count *
2176- !!(q->rate_flags &
2177- CAKE_FLAG_INGRESS))) ||
2178- !flow->head)
2179 break;2184 break;
2180 2185 
2181 /* drop this packet, get another one */2186 /* drop this packet, get another one */
@@ -2189,7 +2194,7 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch)
2189 b->tin_dropped++;2194 b->tin_dropped++;
2190 qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(skb));2195 qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(skb));
2191 qdisc_qstats_drop(sch);2196 qdisc_qstats_drop(sch);
2192- kfree_skb(skb);2197+ kfree_skb_reason(skb, reason);
2193 if (q->rate_flags & CAKE_FLAG_INGRESS)2198 if (q->rate_flags & CAKE_FLAG_INGRESS)
2194 goto retry;2199 goto retry;
2195 }2200 }
@@ -2585,6 +2590,8 @@ static int cake_change(struct Qdisc *sch, struct nlattr *opt,
2585{2590{
2586 struct cake_sched_data *q = qdisc_priv(sch);2591 struct cake_sched_data *q = qdisc_priv(sch);
2587 struct nlattr *tb[TCA_CAKE_MAX + 1];2592 struct nlattr *tb[TCA_CAKE_MAX + 1];
2593+ u16 rate_flags;
2594+ u8 flow_mode;
2588 int err;2595 int err;
2589 2596 
2590 err = nla_parse_nested_deprecated(tb, TCA_CAKE_MAX, opt, cake_policy,2597 err = nla_parse_nested_deprecated(tb, TCA_CAKE_MAX, opt, cake_policy,
@@ -2592,10 +2599,11 @@ static int cake_change(struct Qdisc *sch, struct nlattr *opt,
2592 if (err < 0)2599 if (err < 0)
2593 return err;2600 return err;
2594 2601 
2602+ flow_mode = q->flow_mode;
2595 if (tb[TCA_CAKE_NAT]) {2603 if (tb[TCA_CAKE_NAT]) {
2596#if IS_ENABLED(CONFIG_NF_CONNTRACK)2604#if IS_ENABLED(CONFIG_NF_CONNTRACK)
2597- q->flow_mode &= ~CAKE_FLOW_NAT_FLAG;2605+ flow_mode &= ~CAKE_FLOW_NAT_FLAG;
2598- q->flow_mode |= CAKE_FLOW_NAT_FLAG *2606+ flow_mode |= CAKE_FLOW_NAT_FLAG *
2599 !!nla_get_u32(tb[TCA_CAKE_NAT]);2607 !!nla_get_u32(tb[TCA_CAKE_NAT]);
2600#else2608#else
2601 NL_SET_ERR_MSG_ATTR(extack, tb[TCA_CAKE_NAT],2609 NL_SET_ERR_MSG_ATTR(extack, tb[TCA_CAKE_NAT],
@@ -2605,29 +2613,34 @@ static int cake_change(struct Qdisc *sch, struct nlattr *opt,
2605 }2613 }
2606 2614 
2607 if (tb[TCA_CAKE_BASE_RATE64])2615 if (tb[TCA_CAKE_BASE_RATE64])
2608- q->rate_bps = nla_get_u64(tb[TCA_CAKE_BASE_RATE64]);2616+ WRITE_ONCE(q->rate_bps,
2617+ nla_get_u64(tb[TCA_CAKE_BASE_RATE64]));
2609 2618 
2610 if (tb[TCA_CAKE_DIFFSERV_MODE])2619 if (tb[TCA_CAKE_DIFFSERV_MODE])
2611- q->tin_mode = nla_get_u32(tb[TCA_CAKE_DIFFSERV_MODE]);2620+ WRITE_ONCE(q->tin_mode,
2621+ nla_get_u32(tb[TCA_CAKE_DIFFSERV_MODE]));
2612 2622 
2623+ rate_flags = q->rate_flags;
2613 if (tb[TCA_CAKE_WASH]) {2624 if (tb[TCA_CAKE_WASH]) {
2614 if (!!nla_get_u32(tb[TCA_CAKE_WASH]))2625 if (!!nla_get_u32(tb[TCA_CAKE_WASH]))
2615- q->rate_flags |= CAKE_FLAG_WASH;2626+ rate_flags |= CAKE_FLAG_WASH;
2616 else2627 else
2617- q->rate_flags &= ~CAKE_FLAG_WASH;2628+ rate_flags &= ~CAKE_FLAG_WASH;
2618 }2629 }
2619 2630 
2620 if (tb[TCA_CAKE_FLOW_MODE])2631 if (tb[TCA_CAKE_FLOW_MODE])
2621- q->flow_mode = ((q->flow_mode & CAKE_FLOW_NAT_FLAG) |2632+ flow_mode = ((flow_mode & CAKE_FLOW_NAT_FLAG) |
2622 (nla_get_u32(tb[TCA_CAKE_FLOW_MODE]) &2633 (nla_get_u32(tb[TCA_CAKE_FLOW_MODE]) &
2623 CAKE_FLOW_MASK));2634 CAKE_FLOW_MASK));
2624 2635 
2625 if (tb[TCA_CAKE_ATM])2636 if (tb[TCA_CAKE_ATM])
2626- q->atm_mode = nla_get_u32(tb[TCA_CAKE_ATM]);2637+ WRITE_ONCE(q->atm_mode,
2638+ nla_get_u32(tb[TCA_CAKE_ATM]));
2627 2639 
2628 if (tb[TCA_CAKE_OVERHEAD]) {2640 if (tb[TCA_CAKE_OVERHEAD]) {
2629- q->rate_overhead = nla_get_s32(tb[TCA_CAKE_OVERHEAD]);2641+ WRITE_ONCE(q->rate_overhead,
2630- q->rate_flags |= CAKE_FLAG_OVERHEAD;2642+ nla_get_s32(tb[TCA_CAKE_OVERHEAD]));
2643+ rate_flags |= CAKE_FLAG_OVERHEAD;
2631 2644 
2632 q->max_netlen = 0;2645 q->max_netlen = 0;
2633 q->max_adjlen = 0;2646 q->max_adjlen = 0;
@@ -2636,7 +2649,7 @@ static int cake_change(struct Qdisc *sch, struct nlattr *opt,
2636 }2649 }
2637 2650 
2638 if (tb[TCA_CAKE_RAW]) {2651 if (tb[TCA_CAKE_RAW]) {
2639- q->rate_flags &= ~CAKE_FLAG_OVERHEAD;2652+ rate_flags &= ~CAKE_FLAG_OVERHEAD;
2640 2653 
2641 q->max_netlen = 0;2654 q->max_netlen = 0;
2642 q->max_adjlen = 0;2655 q->max_adjlen = 0;
@@ -2645,54 +2658,58 @@ static int cake_change(struct Qdisc *sch, struct nlattr *opt,
2645 }2658 }
2646 2659 
2647 if (tb[TCA_CAKE_MPU])2660 if (tb[TCA_CAKE_MPU])
2648- q->rate_mpu = nla_get_u32(tb[TCA_CAKE_MPU]);2661+ WRITE_ONCE(q->rate_mpu,
2662+ nla_get_u32(tb[TCA_CAKE_MPU]));
2649 2663 
2650 if (tb[TCA_CAKE_RTT]) {2664 if (tb[TCA_CAKE_RTT]) {
2651- q->interval = nla_get_u32(tb[TCA_CAKE_RTT]);2665+ u32 interval = nla_get_u32(tb[TCA_CAKE_RTT]);
2652 2666 
2653- if (!q->interval)2667+ WRITE_ONCE(q->interval, max(interval, 1U));
2654- q->interval = 1;
2655 }2668 }
2656 2669 
2657 if (tb[TCA_CAKE_TARGET]) {2670 if (tb[TCA_CAKE_TARGET]) {
2658- q->target = nla_get_u32(tb[TCA_CAKE_TARGET]);2671+ u32 target = nla_get_u32(tb[TCA_CAKE_TARGET]);
2659 2672 
2660- if (!q->target)2673+ WRITE_ONCE(q->target, max(target, 1U));
2661- q->target = 1;
2662 }2674 }
2663 2675 
2664 if (tb[TCA_CAKE_AUTORATE]) {2676 if (tb[TCA_CAKE_AUTORATE]) {
2665 if (!!nla_get_u32(tb[TCA_CAKE_AUTORATE]))2677 if (!!nla_get_u32(tb[TCA_CAKE_AUTORATE]))
2666- q->rate_flags |= CAKE_FLAG_AUTORATE_INGRESS;2678+ rate_flags |= CAKE_FLAG_AUTORATE_INGRESS;
2667 else2679 else
2668- q->rate_flags &= ~CAKE_FLAG_AUTORATE_INGRESS;2680+ rate_flags &= ~CAKE_FLAG_AUTORATE_INGRESS;
2669 }2681 }
2670 2682 
2671 if (tb[TCA_CAKE_INGRESS]) {2683 if (tb[TCA_CAKE_INGRESS]) {
2672 if (!!nla_get_u32(tb[TCA_CAKE_INGRESS]))2684 if (!!nla_get_u32(tb[TCA_CAKE_INGRESS]))
2673- q->rate_flags |= CAKE_FLAG_INGRESS;2685+ rate_flags |= CAKE_FLAG_INGRESS;
2674 else2686 else
2675- q->rate_flags &= ~CAKE_FLAG_INGRESS;2687+ rate_flags &= ~CAKE_FLAG_INGRESS;
2676 }2688 }
2677 2689 
2678 if (tb[TCA_CAKE_ACK_FILTER])2690 if (tb[TCA_CAKE_ACK_FILTER])
2679- q->ack_filter = nla_get_u32(tb[TCA_CAKE_ACK_FILTER]);2691+ WRITE_ONCE(q->ack_filter,
2692+ nla_get_u32(tb[TCA_CAKE_ACK_FILTER]));
2680 2693 
2681 if (tb[TCA_CAKE_MEMORY])2694 if (tb[TCA_CAKE_MEMORY])
2682- q->buffer_config_limit = nla_get_u32(tb[TCA_CAKE_MEMORY]);2695+ WRITE_ONCE(q->buffer_config_limit,
2696+ nla_get_u32(tb[TCA_CAKE_MEMORY]));
2683 2697 
2684 if (tb[TCA_CAKE_SPLIT_GSO]) {2698 if (tb[TCA_CAKE_SPLIT_GSO]) {
2685 if (!!nla_get_u32(tb[TCA_CAKE_SPLIT_GSO]))2699 if (!!nla_get_u32(tb[TCA_CAKE_SPLIT_GSO]))
2686- q->rate_flags |= CAKE_FLAG_SPLIT_GSO;2700+ rate_flags |= CAKE_FLAG_SPLIT_GSO;
2687 else2701 else
2688- q->rate_flags &= ~CAKE_FLAG_SPLIT_GSO;2702+ rate_flags &= ~CAKE_FLAG_SPLIT_GSO;
2689 }2703 }
2690 2704 
2691 if (tb[TCA_CAKE_FWMARK]) {2705 if (tb[TCA_CAKE_FWMARK]) {
2692- q->fwmark_mask = nla_get_u32(tb[TCA_CAKE_FWMARK]);2706+ WRITE_ONCE(q->fwmark_mask, nla_get_u32(tb[TCA_CAKE_FWMARK]));
2693- q->fwmark_shft = q->fwmark_mask ? __ffs(q->fwmark_mask) : 0;2707+ WRITE_ONCE(q->fwmark_shft,
2708+ q->fwmark_mask ? __ffs(q->fwmark_mask) : 0);
2694 }2709 }
2695 2710 
2711+ WRITE_ONCE(q->rate_flags, rate_flags);
2712+ WRITE_ONCE(q->flow_mode, flow_mode);
2696 if (q->tins) {2713 if (q->tins) {
2697 sch_tree_lock(sch);2714 sch_tree_lock(sch);
2698 cake_reconfigure(sch);2715 cake_reconfigure(sch);
@@ -2787,68 +2804,72 @@ static int cake_dump(struct Qdisc *sch, struct sk_buff *skb)
2787{2804{
2788 struct cake_sched_data *q = qdisc_priv(sch);2805 struct cake_sched_data *q = qdisc_priv(sch);
2789 struct nlattr *opts;2806 struct nlattr *opts;
2807+ u16 rate_flags;
2808+ u8 flow_mode;
2790 2809 
2791 opts = nla_nest_start_noflag(skb, TCA_OPTIONS);2810 opts = nla_nest_start_noflag(skb, TCA_OPTIONS);
2792 if (!opts)2811 if (!opts)
2793 goto nla_put_failure;2812 goto nla_put_failure;
2794 2813 
2795- if (nla_put_u64_64bit(skb, TCA_CAKE_BASE_RATE64, q->rate_bps,2814+ if (nla_put_u64_64bit(skb, TCA_CAKE_BASE_RATE64,
2796- TCA_CAKE_PAD))2815+ READ_ONCE(q->rate_bps), TCA_CAKE_PAD))
2797 goto nla_put_failure;2816 goto nla_put_failure;
2798 2817 
2799- if (nla_put_u32(skb, TCA_CAKE_FLOW_MODE,2818+ flow_mode = READ_ONCE(q->flow_mode);
2800- q->flow_mode & CAKE_FLOW_MASK))2819+ if (nla_put_u32(skb, TCA_CAKE_FLOW_MODE, flow_mode & CAKE_FLOW_MASK))
2801 goto nla_put_failure;2820 goto nla_put_failure;
2802 2821 
2803- if (nla_put_u32(skb, TCA_CAKE_RTT, q->interval))2822+ if (nla_put_u32(skb, TCA_CAKE_RTT, READ_ONCE(q->interval)))
2804 goto nla_put_failure;2823 goto nla_put_failure;
2805 2824 
2806- if (nla_put_u32(skb, TCA_CAKE_TARGET, q->target))2825+ if (nla_put_u32(skb, TCA_CAKE_TARGET, READ_ONCE(q->target)))
2807 goto nla_put_failure;2826 goto nla_put_failure;
2808 2827 
2809- if (nla_put_u32(skb, TCA_CAKE_MEMORY, q->buffer_config_limit))2828+ if (nla_put_u32(skb, TCA_CAKE_MEMORY,
2829+ READ_ONCE(q->buffer_config_limit)))
2810 goto nla_put_failure;2830 goto nla_put_failure;
2811 2831 
2832+ rate_flags = READ_ONCE(q->rate_flags);
2812 if (nla_put_u32(skb, TCA_CAKE_AUTORATE,2833 if (nla_put_u32(skb, TCA_CAKE_AUTORATE,
2813- !!(q->rate_flags & CAKE_FLAG_AUTORATE_INGRESS)))2834+ !!(rate_flags & CAKE_FLAG_AUTORATE_INGRESS)))
2814 goto nla_put_failure;2835 goto nla_put_failure;
2815 2836 
2816 if (nla_put_u32(skb, TCA_CAKE_INGRESS,2837 if (nla_put_u32(skb, TCA_CAKE_INGRESS,
2817- !!(q->rate_flags & CAKE_FLAG_INGRESS)))2838+ !!(rate_flags & CAKE_FLAG_INGRESS)))
2818 goto nla_put_failure;2839 goto nla_put_failure;
2819 2840 
2820- if (nla_put_u32(skb, TCA_CAKE_ACK_FILTER, q->ack_filter))2841+ if (nla_put_u32(skb, TCA_CAKE_ACK_FILTER, READ_ONCE(q->ack_filter)))
2821 goto nla_put_failure;2842 goto nla_put_failure;
2822 2843 
2823 if (nla_put_u32(skb, TCA_CAKE_NAT,2844 if (nla_put_u32(skb, TCA_CAKE_NAT,
2824- !!(q->flow_mode & CAKE_FLOW_NAT_FLAG)))2845+ !!(flow_mode & CAKE_FLOW_NAT_FLAG)))
2825 goto nla_put_failure;2846 goto nla_put_failure;
2826 2847 
2827- if (nla_put_u32(skb, TCA_CAKE_DIFFSERV_MODE, q->tin_mode))2848+ if (nla_put_u32(skb, TCA_CAKE_DIFFSERV_MODE, READ_ONCE(q->tin_mode)))
2828 goto nla_put_failure;2849 goto nla_put_failure;
2829 2850 
2830 if (nla_put_u32(skb, TCA_CAKE_WASH,2851 if (nla_put_u32(skb, TCA_CAKE_WASH,
2831- !!(q->rate_flags & CAKE_FLAG_WASH)))2852+ !!(rate_flags & CAKE_FLAG_WASH)))
2832 goto nla_put_failure;2853 goto nla_put_failure;
2833 2854 
2834- if (nla_put_u32(skb, TCA_CAKE_OVERHEAD, q->rate_overhead))2855+ if (nla_put_u32(skb, TCA_CAKE_OVERHEAD, READ_ONCE(q->rate_overhead)))
2835 goto nla_put_failure;2856 goto nla_put_failure;
2836 2857 
2837- if (!(q->rate_flags & CAKE_FLAG_OVERHEAD))2858+ if (!(rate_flags & CAKE_FLAG_OVERHEAD))
2838 if (nla_put_u32(skb, TCA_CAKE_RAW, 0))2859 if (nla_put_u32(skb, TCA_CAKE_RAW, 0))
2839 goto nla_put_failure;2860 goto nla_put_failure;
2840 2861 
2841- if (nla_put_u32(skb, TCA_CAKE_ATM, q->atm_mode))2862+ if (nla_put_u32(skb, TCA_CAKE_ATM, READ_ONCE(q->atm_mode)))
2842 goto nla_put_failure;2863 goto nla_put_failure;
2843 2864 
2844- if (nla_put_u32(skb, TCA_CAKE_MPU, q->rate_mpu))2865+ if (nla_put_u32(skb, TCA_CAKE_MPU, READ_ONCE(q->rate_mpu)))
2845 goto nla_put_failure;2866 goto nla_put_failure;
2846 2867 
2847 if (nla_put_u32(skb, TCA_CAKE_SPLIT_GSO,2868 if (nla_put_u32(skb, TCA_CAKE_SPLIT_GSO,
2848- !!(q->rate_flags & CAKE_FLAG_SPLIT_GSO)))2869+ !!(rate_flags & CAKE_FLAG_SPLIT_GSO)))
2849 goto nla_put_failure;2870 goto nla_put_failure;
2850 2871 
2851- if (nla_put_u32(skb, TCA_CAKE_FWMARK, q->fwmark_mask))2872+ if (nla_put_u32(skb, TCA_CAKE_FWMARK, READ_ONCE(q->fwmark_mask)))
2852 goto nla_put_failure;2873 goto nla_put_failure;
2853 2874 
2854 return nla_nest_end(skb, opts);2875 return nla_nest_end(skb, opts);
@@ -3116,6 +3137,7 @@ static struct Qdisc_ops cake_qdisc_ops __read_mostly = {
3116 .dump_stats = cake_dump_stats,3137 .dump_stats = cake_dump_stats,
3117 .owner = THIS_MODULE,3138 .owner = THIS_MODULE,
3118};3139};
3140+MODULE_ALIAS_NET_SCH("cake");
3119 3141 
3120static int __init cake_module_init(void)3142static int __init cake_module_init(void)
3121{3143{
Msecurity/integrity/ima/ima_policy.c+1-1
@@ -672,7 +672,7 @@ static bool ima_match_rules(struct ima_rule_entry *rule,
672 goto retry;672 goto retry;
673 }673 }
674 }674 }
675- if (!rc) {675+ if (rc <= 0) {
676 result = false;676 result = false;
677 goto out;677 goto out;
678 }678 }
Msound/firewire/dice/dice-extension.c+2-2
@@ -116,7 +116,7 @@ static int detect_stream_formats(struct snd_dice *dice, u64 section_addr)
116 break;116 break;
117 117 
118 base_offset += EXT_APP_STREAM_ENTRIES;118 base_offset += EXT_APP_STREAM_ENTRIES;
119- stream_count = be32_to_cpu(reg[0]);119+ stream_count = min_t(unsigned int, be32_to_cpu(reg[0]), MAX_STREAMS);
120 err = read_stream_entries(dice, section_addr, base_offset,120 err = read_stream_entries(dice, section_addr, base_offset,
121 stream_count, mode,121 stream_count, mode,
122 dice->tx_pcm_chs,122 dice->tx_pcm_chs,
@@ -125,7 +125,7 @@ static int detect_stream_formats(struct snd_dice *dice, u64 section_addr)
125 break;125 break;
126 126 
127 base_offset += stream_count * EXT_APP_STREAM_ENTRY_SIZE;127 base_offset += stream_count * EXT_APP_STREAM_ENTRY_SIZE;
128- stream_count = be32_to_cpu(reg[1]);128+ stream_count = min_t(unsigned int, be32_to_cpu(reg[1]), MAX_STREAMS);
129 err = read_stream_entries(dice, section_addr, base_offset,129 err = read_stream_entries(dice, section_addr, base_offset,
130 stream_count,130 stream_count,
131 mode, dice->rx_pcm_chs,131 mode, dice->rx_pcm_chs,
Msound/firewire/motu/motu-hwdep.c+1-1
@@ -75,7 +75,7 @@ static long hwdep_read(struct snd_hwdep *hwdep, char __user *buf, long count,
75 while (consumed < count &&75 while (consumed < count &&
76 snd_motu_register_dsp_message_parser_copy_event(motu, &ev)) {76 snd_motu_register_dsp_message_parser_copy_event(motu, &ev)) {
77 ptr = (u32 __user *)(buf + consumed);77 ptr = (u32 __user *)(buf + consumed);
78- if (put_user(ev, ptr))78+ if (consumed + sizeof(ev) > count || put_user(ev, ptr))
79 return -EFAULT;79 return -EFAULT;
80 consumed += sizeof(ev);80 consumed += sizeof(ev);
81 }81 }
Msound/isa/wavefront/wavefront_synth.c+109-107
@@ -116,7 +116,7 @@ MODULE_PARM_DESC(osrun_time, "how many seconds to wait for the ICS2115 OS");
116 116 
117#define DPRINT(cond, ...) \117#define DPRINT(cond, ...) \
118 if ((dev->debug & (cond)) == (cond)) { \118 if ((dev->debug & (cond)) == (cond)) { \
119- snd_printk (__VA_ARGS__); \119+ pr_debug(__VA_ARGS__); \
120 }120 }
121#else121#else
122#define DPRINT(cond, args...)122#define DPRINT(cond, args...)
@@ -341,7 +341,7 @@ snd_wavefront_cmd (snd_wavefront_t *dev,
341 341 
342 wfcmd = wavefront_get_command(cmd);342 wfcmd = wavefront_get_command(cmd);
343 if (!wfcmd) {343 if (!wfcmd) {
344- snd_printk ("command 0x%x not supported.\n",344+ dev_err(dev->card->dev, "command 0x%x not supported.\n",
345 cmd);345 cmd);
346 return 1;346 return 1;
347 }347 }
@@ -623,7 +623,7 @@ wavefront_get_sample_status (snd_wavefront_t *dev, int assume_rom)
623 /* check sample status */623 /* check sample status */
624 624
625 if (snd_wavefront_cmd (dev, WFC_GET_NSAMPLES, rbuf, wbuf)) {625 if (snd_wavefront_cmd (dev, WFC_GET_NSAMPLES, rbuf, wbuf)) {
626- snd_printk ("cannot request sample count.\n");626+ dev_err(dev->card->dev, "cannot request sample count.\n");
627 return -1;627 return -1;
628 } 628 }
629 629
@@ -635,8 +635,8 @@ wavefront_get_sample_status (snd_wavefront_t *dev, int assume_rom)
635 wbuf[1] = i >> 7;635 wbuf[1] = i >> 7;
636 636 
637 if (snd_wavefront_cmd (dev, WFC_IDENTIFY_SAMPLE_TYPE, rbuf, wbuf)) {637 if (snd_wavefront_cmd (dev, WFC_IDENTIFY_SAMPLE_TYPE, rbuf, wbuf)) {
638- snd_printk(KERN_WARNING "cannot identify sample "638+ dev_warn(dev->card->dev,
639- "type of slot %d\n", i);639+ "cannot identify sample type of slot %d\n", i);
640 dev->sample_status[i] = WF_ST_EMPTY;640 dev->sample_status[i] = WF_ST_EMPTY;
641 continue;641 continue;
642 }642 }
@@ -661,9 +661,9 @@ wavefront_get_sample_status (snd_wavefront_t *dev, int assume_rom)
661 break;661 break;
662 662 
663 default:663 default:
664- snd_printk ("unknown sample type for "664+ dev_err(dev->card->dev,
665- "slot %d (0x%x)\n", 665+ "unknown sample type for slot %d (0x%x)\n",
666- i, rbuf[0]);666+ i, rbuf[0]);
667 }667 }
668 668 
669 if (rbuf[0] != WF_ST_EMPTY) {669 if (rbuf[0] != WF_ST_EMPTY) {
@@ -671,9 +671,10 @@ wavefront_get_sample_status (snd_wavefront_t *dev, int assume_rom)
671 } 671 }
672 }672 }
673 673 
674- snd_printk ("%d samples used (%d real, %d aliases, %d multi), "674+ dev_info(dev->card->dev,
675- "%d empty\n", dev->samples_used, sc_real, sc_alias, sc_multi,675+ "%d samples used (%d real, %d aliases, %d multi), %d empty\n",
676- WF_MAX_SAMPLE - dev->samples_used);676+ dev->samples_used, sc_real, sc_alias, sc_multi,
677+ WF_MAX_SAMPLE - dev->samples_used);
677 678 
678 679 
679 return (0);680 return (0);
@@ -706,8 +707,8 @@ wavefront_get_patch_status (snd_wavefront_t *dev)
706 } else if (x == 3) { /* Bad patch number */707 } else if (x == 3) { /* Bad patch number */
707 dev->patch_status[i] = 0;708 dev->patch_status[i] = 0;
708 } else {709 } else {
709- snd_printk ("upload patch "710+ dev_err(dev->card->dev,
710- "error 0x%x\n", x);711+ "upload patch error 0x%x\n", x);
711 dev->patch_status[i] = 0;712 dev->patch_status[i] = 0;
712 return 1;713 return 1;
713 }714 }
@@ -724,7 +725,8 @@ wavefront_get_patch_status (snd_wavefront_t *dev)
724 }725 }
725 726
726 }727 }
727- snd_printk ("%d patch slots filled, %d in use\n", cnt, cnt2);728+ dev_info(dev->card->dev, "%d patch slots filled, %d in use\n",
729+ cnt, cnt2);
728 730 
729 return (0);731 return (0);
730}732}
@@ -760,8 +762,8 @@ wavefront_get_program_status (snd_wavefront_t *dev)
760 } else if (x == 1) { /* Bad program number */762 } else if (x == 1) { /* Bad program number */
761 dev->prog_status[i] = 0;763 dev->prog_status[i] = 0;
762 } else {764 } else {
763- snd_printk ("upload program "765+ dev_err(dev->card->dev,
764- "error 0x%x\n", x);766+ "upload program error 0x%x\n", x);
765 dev->prog_status[i] = 0;767 dev->prog_status[i] = 0;
766 }768 }
767 }769 }
@@ -772,7 +774,7 @@ wavefront_get_program_status (snd_wavefront_t *dev)
772 }774 }
773 }775 }
774 776 
775- snd_printk ("%d programs slots in use\n", cnt);777+ dev_info(dev->card->dev, "%d programs slots in use\n", cnt);
776 778 
777 return (0);779 return (0);
778}780}
@@ -796,7 +798,7 @@ wavefront_send_patch (snd_wavefront_t *dev, wavefront_patch_info *header)
796 munge_buf ((unsigned char *)&header->hdr.p, bptr, WF_PATCH_BYTES);798 munge_buf ((unsigned char *)&header->hdr.p, bptr, WF_PATCH_BYTES);
797 799
798 if (snd_wavefront_cmd (dev, WFC_DOWNLOAD_PATCH, NULL, buf)) {800 if (snd_wavefront_cmd (dev, WFC_DOWNLOAD_PATCH, NULL, buf)) {
799- snd_printk ("download patch failed\n");801+ dev_err(dev->card->dev, "download patch failed\n");
800 return -EIO;802 return -EIO;
801 }803 }
802 804 
@@ -837,7 +839,7 @@ wavefront_send_program (snd_wavefront_t *dev, wavefront_patch_info *header)
837 munge_buf ((unsigned char *)&header->hdr.pr, &buf[1], WF_PROGRAM_BYTES);839 munge_buf ((unsigned char *)&header->hdr.pr, &buf[1], WF_PROGRAM_BYTES);
838 840
839 if (snd_wavefront_cmd (dev, WFC_DOWNLOAD_PROGRAM, NULL, buf)) {841 if (snd_wavefront_cmd (dev, WFC_DOWNLOAD_PROGRAM, NULL, buf)) {
840- snd_printk ("download patch failed\n"); 842+ dev_err(dev->card->dev, "download patch failed\n");
841 return -EIO;843 return -EIO;
842 }844 }
843 845 
@@ -851,7 +853,7 @@ wavefront_freemem (snd_wavefront_t *dev)
851 char rbuf[8];853 char rbuf[8];
852 854 
853 if (snd_wavefront_cmd (dev, WFC_REPORT_FREE_MEMORY, rbuf, NULL)) {855 if (snd_wavefront_cmd (dev, WFC_REPORT_FREE_MEMORY, rbuf, NULL)) {
854- snd_printk ("can't get memory stats.\n");856+ dev_err(dev->card->dev, "can't get memory stats.\n");
855 return -1;857 return -1;
856 } else {858 } else {
857 return demunge_int32 (rbuf, 4);859 return demunge_int32 (rbuf, 4);
@@ -901,7 +903,7 @@ wavefront_send_sample (snd_wavefront_t *dev,
901 x = wavefront_find_free_sample(dev);903 x = wavefront_find_free_sample(dev);
902 if (x < 0)904 if (x < 0)
903 return -ENOMEM;905 return -ENOMEM;
904- snd_printk ("unspecified sample => %d\n", x);906+ dev_info(dev->card->dev, "unspecified sample => %d\n", x);
905 header->number = x;907 header->number = x;
906 }908 }
907 909 
@@ -935,9 +937,9 @@ wavefront_send_sample (snd_wavefront_t *dev,
935 937 
936 if (dev->rom_samples_rdonly) {938 if (dev->rom_samples_rdonly) {
937 if (dev->sample_status[header->number] & WF_SLOT_ROM) {939 if (dev->sample_status[header->number] & WF_SLOT_ROM) {
938- snd_printk ("sample slot %d "940+ dev_err(dev->card->dev,
939- "write protected\n",941+ "sample slot %d write protected\n",
940- header->number);942+ header->number);
941 return -EACCES;943 return -EACCES;
942 }944 }
943 }945 }
@@ -948,10 +950,10 @@ wavefront_send_sample (snd_wavefront_t *dev,
948 if (header->size) {950 if (header->size) {
949 dev->freemem = wavefront_freemem (dev);951 dev->freemem = wavefront_freemem (dev);
950 952 
951- if (dev->freemem < (int)header->size) {953+ if (dev->freemem < 0 || dev->freemem < header->size) {
952- snd_printk ("insufficient memory to "954+ dev_err(dev->card->dev,
953- "load %d byte sample.\n",955+ "insufficient memory to load %u byte sample.\n",
954- header->size);956+ header->size);
955 return -ENOMEM;957 return -ENOMEM;
956 }958 }
957 959
@@ -960,8 +962,8 @@ wavefront_send_sample (snd_wavefront_t *dev,
960 skip = WF_GET_CHANNEL(&header->hdr.s);962 skip = WF_GET_CHANNEL(&header->hdr.s);
961 963 
962 if (skip > 0 && header->hdr.s.SampleResolution != LINEAR_16BIT) {964 if (skip > 0 && header->hdr.s.SampleResolution != LINEAR_16BIT) {
963- snd_printk ("channel selection only "965+ dev_err(dev->card->dev,
964- "possible on 16-bit samples");966+ "channel selection only possible on 16-bit samples");
965 return -EINVAL;967 return -EINVAL;
966 }968 }
967 969 
@@ -1057,8 +1059,8 @@ wavefront_send_sample (snd_wavefront_t *dev,
1057 header->size ?1059 header->size ?
1058 WFC_DOWNLOAD_SAMPLE : WFC_DOWNLOAD_SAMPLE_HEADER,1060 WFC_DOWNLOAD_SAMPLE : WFC_DOWNLOAD_SAMPLE_HEADER,
1059 NULL, sample_hdr)) {1061 NULL, sample_hdr)) {
1060- snd_printk ("sample %sdownload refused.\n",1062+ dev_err(dev->card->dev, "sample %sdownload refused.\n",
1061- header->size ? "" : "header ");1063+ header->size ? "" : "header ");
1062 return -EIO;1064 return -EIO;
1063 }1065 }
1064 1066 
@@ -1083,8 +1085,8 @@ wavefront_send_sample (snd_wavefront_t *dev,
1083 }1085 }
1084 1086 
1085 if (snd_wavefront_cmd (dev, WFC_DOWNLOAD_BLOCK, NULL, NULL)) {1087 if (snd_wavefront_cmd (dev, WFC_DOWNLOAD_BLOCK, NULL, NULL)) {
1086- snd_printk ("download block "1088+ dev_err(dev->card->dev,
1087- "request refused.\n");1089+ "download block request refused.\n");
1088 return -EIO;1090 return -EIO;
1089 }1091 }
1090 1092 
@@ -1145,13 +1147,13 @@ wavefront_send_sample (snd_wavefront_t *dev,
1145 dma_ack = wavefront_read(dev);1147 dma_ack = wavefront_read(dev);
1146 if (dma_ack != WF_DMA_ACK) {1148 if (dma_ack != WF_DMA_ACK) {
1147 if (dma_ack == -1) {1149 if (dma_ack == -1) {
1148- snd_printk ("upload sample "1150+ dev_err(dev->card->dev,
1149- "DMA ack timeout\n");1151+ "upload sample DMA ack timeout\n");
1150 return -EIO;1152 return -EIO;
1151 } else {1153 } else {
1152- snd_printk ("upload sample "1154+ dev_err(dev->card->dev,
1153- "DMA ack error 0x%x\n",1155+ "upload sample DMA ack error 0x%x\n",
1154- dma_ack);1156+ dma_ack);
1155 return -EIO;1157 return -EIO;
1156 }1158 }
1157 }1159 }
@@ -1195,7 +1197,7 @@ wavefront_send_alias (snd_wavefront_t *dev, wavefront_patch_info *header)
1195 munge_int32 (*(&header->hdr.a.FrequencyBias+1), &alias_hdr[23], 2);1197 munge_int32 (*(&header->hdr.a.FrequencyBias+1), &alias_hdr[23], 2);
1196 1198 
1197 if (snd_wavefront_cmd (dev, WFC_DOWNLOAD_SAMPLE_ALIAS, NULL, alias_hdr)) {1199 if (snd_wavefront_cmd (dev, WFC_DOWNLOAD_SAMPLE_ALIAS, NULL, alias_hdr)) {
1198- snd_printk ("download alias failed.\n");1200+ dev_err(dev->card->dev, "download alias failed.\n");
1199 return -EIO;1201 return -EIO;
1200 }1202 }
1201 1203 
@@ -1248,7 +1250,7 @@ wavefront_send_multisample (snd_wavefront_t *dev, wavefront_patch_info *header)
1248 if (snd_wavefront_cmd (dev, WFC_DOWNLOAD_MULTISAMPLE, 1250 if (snd_wavefront_cmd (dev, WFC_DOWNLOAD_MULTISAMPLE,
1249 (unsigned char *) (long) ((num_samples*2)+3),1251 (unsigned char *) (long) ((num_samples*2)+3),
1250 msample_hdr)) {1252 msample_hdr)) {
1251- snd_printk ("download of multisample failed.\n");1253+ dev_err(dev->card->dev, "download of multisample failed.\n");
1252 kfree(msample_hdr);1254 kfree(msample_hdr);
1253 return -EIO;1255 return -EIO;
1254 }1256 }
@@ -1271,7 +1273,7 @@ wavefront_fetch_multisample (snd_wavefront_t *dev,
1271 munge_int32 (header->number, number, 2);1273 munge_int32 (header->number, number, 2);
1272 1274
1273 if (snd_wavefront_cmd (dev, WFC_UPLOAD_MULTISAMPLE, log_ns, number)) {1275 if (snd_wavefront_cmd (dev, WFC_UPLOAD_MULTISAMPLE, log_ns, number)) {
1274- snd_printk ("upload multisample failed.\n");1276+ dev_err(dev->card->dev, "upload multisample failed.\n");
1275 return -EIO;1277 return -EIO;
1276 }1278 }
1277 1279
@@ -1290,16 +1292,16 @@ wavefront_fetch_multisample (snd_wavefront_t *dev,
1290 1292
1291 val = wavefront_read(dev);1293 val = wavefront_read(dev);
1292 if (val == -1) {1294 if (val == -1) {
1293- snd_printk ("upload multisample failed "1295+ dev_err(dev->card->dev,
1294- "during sample loop.\n");1296+ "upload multisample failed during sample loop.\n");
1295 return -EIO;1297 return -EIO;
1296 }1298 }
1297 d[0] = val;1299 d[0] = val;
1298 1300 
1299 val = wavefront_read(dev);1301 val = wavefront_read(dev);
1300 if (val == -1) {1302 if (val == -1) {
1301- snd_printk ("upload multisample failed "1303+ dev_err(dev->card->dev,
1302- "during sample loop.\n");1304+ "upload multisample failed during sample loop.\n");
1303 return -EIO;1305 return -EIO;
1304 }1306 }
1305 d[1] = val;1307 d[1] = val;
@@ -1334,7 +1336,7 @@ wavefront_send_drum (snd_wavefront_t *dev, wavefront_patch_info *header)
1334 }1336 }
1335 1337 
1336 if (snd_wavefront_cmd (dev, WFC_DOWNLOAD_EDRUM_PROGRAM, NULL, drumbuf)) {1338 if (snd_wavefront_cmd (dev, WFC_DOWNLOAD_EDRUM_PROGRAM, NULL, drumbuf)) {
1337- snd_printk ("download drum failed.\n");1339+ dev_err(dev->card->dev, "download drum failed.\n");
1338 return -EIO;1340 return -EIO;
1339 }1341 }
1340 1342 
@@ -1352,7 +1354,7 @@ wavefront_find_free_sample (snd_wavefront_t *dev)
1352 return i;1354 return i;
1353 }1355 }
1354 }1356 }
1355- snd_printk ("no free sample slots!\n");1357+ dev_err(dev->card->dev, "no free sample slots!\n");
1356 return -1;1358 return -1;
1357}1359}
1358 1360 
@@ -1368,7 +1370,7 @@ wavefront_find_free_patch (snd_wavefront_t *dev)
1368 return i;1370 return i;
1369 }1371 }
1370 }1372 }
1371- snd_printk ("no free patch slots!\n");1373+ dev_err(dev->card->dev, "no free patch slots!\n");
1372 return -1;1374 return -1;
1373}1375}
1374#endif1376#endif
@@ -1385,7 +1387,7 @@ wavefront_load_patch (snd_wavefront_t *dev, const char __user *addr)
1385 1387 
1386 if (copy_from_user (header, addr, sizeof(wavefront_patch_info) -1388 if (copy_from_user (header, addr, sizeof(wavefront_patch_info) -
1387 sizeof(wavefront_any))) {1389 sizeof(wavefront_any))) {
1388- snd_printk ("bad address for load patch.\n");1390+ dev_err(dev->card->dev, "bad address for load patch.\n");
1389 err = -EFAULT;1391 err = -EFAULT;
1390 goto __error;1392 goto __error;
1391 }1393 }
@@ -1463,8 +1465,8 @@ wavefront_load_patch (snd_wavefront_t *dev, const char __user *addr)
1463 break;1465 break;
1464 1466 
1465 default:1467 default:
1466- snd_printk ("unknown patch type %d.\n",1468+ dev_err(dev->card->dev, "unknown patch type %d.\n",
1467- header->subkey);1469+ header->subkey);
1468 err = -EINVAL;1470 err = -EINVAL;
1469 break;1471 break;
1470 }1472 }
@@ -1527,13 +1529,13 @@ wavefront_synth_control (snd_wavefront_card_t *acard,
1527 switch (wc->cmd) {1529 switch (wc->cmd) {
1528 1530
1529 case WFC_DISABLE_INTERRUPTS:1531 case WFC_DISABLE_INTERRUPTS:
1530- snd_printk ("interrupts disabled.\n");1532+ dev_dbg(dev->card->dev, "interrupts disabled.\n");
1531 outb (0x80|0x20, dev->control_port);1533 outb (0x80|0x20, dev->control_port);
1532 dev->interrupts_are_midi = 1;1534 dev->interrupts_are_midi = 1;
1533 return 0;1535 return 0;
1534 1536 
1535 case WFC_ENABLE_INTERRUPTS:1537 case WFC_ENABLE_INTERRUPTS:
1536- snd_printk ("interrupts enabled.\n");1538+ dev_dbg(dev->card->dev, "interrupts enabled.\n");
1537 outb (0x80|0x40|0x20, dev->control_port);1539 outb (0x80|0x40|0x20, dev->control_port);
1538 dev->interrupts_are_midi = 1;1540 dev->interrupts_are_midi = 1;
1539 return 0;1541 return 0;
@@ -1550,7 +1552,7 @@ wavefront_synth_control (snd_wavefront_card_t *acard,
1550 case WFC_IDENTIFY_SLOT_TYPE:1552 case WFC_IDENTIFY_SLOT_TYPE:
1551 i = wc->wbuf[0] | (wc->wbuf[1] << 7);1553 i = wc->wbuf[0] | (wc->wbuf[1] << 7);
1552 if (i <0 || i >= WF_MAX_SAMPLE) {1554 if (i <0 || i >= WF_MAX_SAMPLE) {
1553- snd_printk ("invalid slot ID %d\n",1555+ dev_err(dev->card->dev, "invalid slot ID %d\n",
1554 i);1556 i);
1555 wc->status = EINVAL;1557 wc->status = EINVAL;
1556 return -EINVAL;1558 return -EINVAL;
@@ -1561,7 +1563,7 @@ wavefront_synth_control (snd_wavefront_card_t *acard,
1561 1563 
1562 case WFC_DEBUG_DRIVER:1564 case WFC_DEBUG_DRIVER:
1563 dev->debug = wc->wbuf[0];1565 dev->debug = wc->wbuf[0];
1564- snd_printk ("debug = 0x%x\n", dev->debug);1566+ dev_dbg(dev->card->dev, "debug = 0x%x\n", dev->debug);
1565 return 0;1567 return 0;
1566 1568 
1567 case WFC_UPLOAD_PATCH:1569 case WFC_UPLOAD_PATCH:
@@ -1578,8 +1580,8 @@ wavefront_synth_control (snd_wavefront_card_t *acard,
1578 return 0;1580 return 0;
1579 1581 
1580 case WFC_UPLOAD_SAMPLE_ALIAS:1582 case WFC_UPLOAD_SAMPLE_ALIAS:
1581- snd_printk ("support for sample alias upload "1583+ dev_err(dev->card->dev,
1582- "being considered.\n");1584+ "support for sample alias upload being considered.\n");
1583 wc->status = EINVAL;1585 wc->status = EINVAL;
1584 return -EINVAL;1586 return -EINVAL;
1585 }1587 }
@@ -1620,9 +1622,8 @@ wavefront_synth_control (snd_wavefront_card_t *acard,
1620 break;1622 break;
1621 1623 
1622 case WFC_UPLOAD_SAMPLE_ALIAS:1624 case WFC_UPLOAD_SAMPLE_ALIAS:
1623- snd_printk ("support for "1625+ dev_err(dev->card->dev,
1624- "sample aliases still "1626+ "support for sample aliases still being considered.\n");
1625- "being considered.\n");
1626 break;1627 break;
1627 1628 
1628 case WFC_VMIDI_OFF:1629 case WFC_VMIDI_OFF:
@@ -1740,10 +1741,10 @@ snd_wavefront_internal_interrupt (snd_wavefront_card_t *card)
1740 return;1741 return;
1741 }1742 }
1742 1743 
1743- spin_lock(&dev->irq_lock);1744+ scoped_guard(spinlock, &dev->irq_lock) {
1744- dev->irq_ok = 1;1745+ dev->irq_ok = 1;
1745- dev->irq_cnt++;1746+ dev->irq_cnt++;
1746- spin_unlock(&dev->irq_lock);1747+ }
1747 wake_up(&dev->interrupt_sleeper);1748 wake_up(&dev->interrupt_sleeper);
1748}1749}
1749 1750 
@@ -1760,7 +1761,7 @@ snd_wavefront_internal_interrupt (snd_wavefront_card_t *card)
1760*/1761*/
1761 1762 
1762static int1763static int
1763-snd_wavefront_interrupt_bits (int irq)1764+snd_wavefront_interrupt_bits(snd_wavefront_t *dev, int irq)
1764 1765 
1765{1766{
1766 int bits;1767 int bits;
@@ -1780,7 +1781,7 @@ snd_wavefront_interrupt_bits (int irq)
1780 break;1781 break;
1781 1782
1782 default:1783 default:
1783- snd_printk ("invalid IRQ %d\n", irq);1784+ dev_err(dev->card->dev, "invalid IRQ %d\n", irq);
1784 bits = -1;1785 bits = -1;
1785 }1786 }
1786 1787 
@@ -1795,11 +1796,11 @@ wavefront_should_cause_interrupt (snd_wavefront_t *dev,
1795 wait_queue_entry_t wait;1796 wait_queue_entry_t wait;
1796 1797 
1797 init_waitqueue_entry(&wait, current);1798 init_waitqueue_entry(&wait, current);
1798- spin_lock_irq(&dev->irq_lock);1799+ scoped_guard(spinlock_irq, &dev->irq_lock) {
1799- add_wait_queue(&dev->interrupt_sleeper, &wait);1800+ add_wait_queue(&dev->interrupt_sleeper, &wait);
1800- dev->irq_ok = 0;1801+ dev->irq_ok = 0;
1801- outb (val,port);1802+ outb(val, port);
1802- spin_unlock_irq(&dev->irq_lock);1803+ }
1803 while (!dev->irq_ok && time_before(jiffies, timeout)) {1804 while (!dev->irq_ok && time_before(jiffies, timeout)) {
1804 schedule_timeout_uninterruptible(1);1805 schedule_timeout_uninterruptible(1);
1805 barrier();1806 barrier();
@@ -1815,7 +1816,7 @@ wavefront_reset_to_cleanliness (snd_wavefront_t *dev)
1815 1816 
1816 /* IRQ already checked */1817 /* IRQ already checked */
1817 1818 
1818- bits = snd_wavefront_interrupt_bits (dev->irq);1819+ bits = snd_wavefront_interrupt_bits(dev, dev->irq);
1819 1820 
1820 /* try reset of port */1821 /* try reset of port */
1821 1822 
@@ -1885,7 +1886,7 @@ wavefront_reset_to_cleanliness (snd_wavefront_t *dev)
1885 */1886 */
1886 1887 
1887 if (!dev->irq_ok) {1888 if (!dev->irq_ok) {
1888- snd_printk ("intr not received after h/w un-reset.\n");1889+ dev_err(dev->card->dev, "intr not received after h/w un-reset.\n");
1889 goto gone_bad;1890 goto gone_bad;
1890 } 1891 }
1891 1892 
@@ -1909,18 +1910,18 @@ wavefront_reset_to_cleanliness (snd_wavefront_t *dev)
1909 dev->data_port, ramcheck_time*HZ);1910 dev->data_port, ramcheck_time*HZ);
1910 1911 
1911 if (!dev->irq_ok) {1912 if (!dev->irq_ok) {
1912- snd_printk ("post-RAM-check interrupt not received.\n");1913+ dev_err(dev->card->dev, "post-RAM-check interrupt not received.\n");
1913 goto gone_bad;1914 goto gone_bad;
1914 } 1915 }
1915 1916 
1916 if (!wavefront_wait (dev, STAT_CAN_READ)) {1917 if (!wavefront_wait (dev, STAT_CAN_READ)) {
1917- snd_printk ("no response to HW version cmd.\n");1918+ dev_err(dev->card->dev, "no response to HW version cmd.\n");
1918 goto gone_bad;1919 goto gone_bad;
1919 }1920 }
1920 1921
1921 hwv[0] = wavefront_read(dev);1922 hwv[0] = wavefront_read(dev);
1922 if (hwv[0] == -1) {1923 if (hwv[0] == -1) {
1923- snd_printk ("board not responding correctly.\n");1924+ dev_err(dev->card->dev, "board not responding correctly.\n");
1924 goto gone_bad;1925 goto gone_bad;
1925 }1926 }
1926 1927 
@@ -1932,11 +1933,11 @@ wavefront_reset_to_cleanliness (snd_wavefront_t *dev)
1932 1933
1933 hwv[0] = wavefront_read(dev);1934 hwv[0] = wavefront_read(dev);
1934 if (hwv[0] == -1) {1935 if (hwv[0] == -1) {
1935- snd_printk ("on-board RAM test failed "1936+ dev_err(dev->card->dev,
1936- "(bad error code).\n");1937+ "on-board RAM test failed (bad error code).\n");
1937 } else {1938 } else {
1938- snd_printk ("on-board RAM test failed "1939+ dev_err(dev->card->dev,
1939- "(error code: 0x%x).\n",1940+ "on-board RAM test failed (error code: 0x%x).\n",
1940 hwv[0]);1941 hwv[0]);
1941 }1942 }
1942 goto gone_bad;1943 goto gone_bad;
@@ -1946,12 +1947,12 @@ wavefront_reset_to_cleanliness (snd_wavefront_t *dev)
1946 1947 
1947 hwv[1] = wavefront_read(dev);1948 hwv[1] = wavefront_read(dev);
1948 if (hwv[1] == -1) {1949 if (hwv[1] == -1) {
1949- snd_printk ("incorrect h/w response.\n");1950+ dev_err(dev->card->dev, "incorrect h/w response.\n");
1950 goto gone_bad;1951 goto gone_bad;
1951 }1952 }
1952 1953 
1953- snd_printk ("hardware version %d.%d\n",1954+ dev_info(dev->card->dev, "hardware version %d.%d\n",
1954- hwv[0], hwv[1]);1955+ hwv[0], hwv[1]);
1955 1956 
1956 return 0;1957 return 0;
1957 1958 
@@ -1971,7 +1972,7 @@ wavefront_download_firmware (snd_wavefront_t *dev, char *path)
1971 1972 
1972 err = request_firmware(&firmware, path, dev->card->dev);1973 err = request_firmware(&firmware, path, dev->card->dev);
1973 if (err < 0) {1974 if (err < 0) {
1974- snd_printk(KERN_ERR "firmware (%s) download failed!!!\n", path);1975+ dev_err(dev->card->dev, "firmware (%s) download failed!!!\n", path);
1975 return 1;1976 return 1;
1976 }1977 }
1977 1978 
@@ -1982,16 +1983,16 @@ wavefront_download_firmware (snd_wavefront_t *dev, char *path)
1982 if (section_length == 0)1983 if (section_length == 0)
1983 break;1984 break;
1984 if (section_length < 0 || section_length > WF_SECTION_MAX) {1985 if (section_length < 0 || section_length > WF_SECTION_MAX) {
1985- snd_printk(KERN_ERR1986+ dev_err(dev->card->dev,
1986- "invalid firmware section length %d\n",1987+ "invalid firmware section length %d\n",
1987- section_length);1988+ section_length);
1988 goto failure;1989 goto failure;
1989 }1990 }
1990 buf++;1991 buf++;
1991 len++;1992 len++;
1992 1993 
1993 if (firmware->size < len + section_length) {1994 if (firmware->size < len + section_length) {
1994- snd_printk(KERN_ERR "firmware section read error.\n");1995+ dev_err(dev->card->dev, "firmware section read error.\n");
1995 goto failure;1996 goto failure;
1996 }1997 }
1997 1998 
@@ -2008,15 +2009,14 @@ wavefront_download_firmware (snd_wavefront_t *dev, char *path)
2008 2009
2009 /* get ACK */2010 /* get ACK */
2010 if (!wavefront_wait(dev, STAT_CAN_READ)) {2011 if (!wavefront_wait(dev, STAT_CAN_READ)) {
2011- snd_printk(KERN_ERR "time out for firmware ACK.\n");2012+ dev_err(dev->card->dev, "time out for firmware ACK.\n");
2012 goto failure;2013 goto failure;
2013 }2014 }
2014 err = inb(dev->data_port);2015 err = inb(dev->data_port);
2015 if (err != WF_ACK) {2016 if (err != WF_ACK) {
2016- snd_printk(KERN_ERR2017+ dev_err(dev->card->dev,
2017- "download of section #%d not "2018+ "download of section #%d not acknowledged, ack = 0x%x\n",
2018- "acknowledged, ack = 0x%x\n",2019+ section_cnt_downloaded + 1, err);
2019- section_cnt_downloaded + 1, err);
2020 goto failure;2020 goto failure;
2021 }2021 }
2022 2022 
@@ -2028,7 +2028,7 @@ wavefront_download_firmware (snd_wavefront_t *dev, char *path)
2028 2028 
2029 failure:2029 failure:
2030 release_firmware(firmware);2030 release_firmware(firmware);
2031- snd_printk(KERN_ERR "firmware download failed!!!\n");2031+ dev_err(dev->card->dev, "firmware download failed!!!\n");
2032 return 1;2032 return 1;
2033}2033}
2034 2034 
@@ -2040,7 +2040,7 @@ wavefront_do_reset (snd_wavefront_t *dev)
2040 char voices[1];2040 char voices[1];
2041 2041 
2042 if (wavefront_reset_to_cleanliness (dev)) {2042 if (wavefront_reset_to_cleanliness (dev)) {
2043- snd_printk ("hw reset failed.\n");2043+ dev_err(dev->card->dev, "hw reset failed.\n");
2044 goto gone_bad;2044 goto gone_bad;
2045 }2045 }
2046 2046 
@@ -2064,7 +2064,7 @@ wavefront_do_reset (snd_wavefront_t *dev)
2064 (osrun_time*HZ));2064 (osrun_time*HZ));
2065 2065 
2066 if (!dev->irq_ok) {2066 if (!dev->irq_ok) {
2067- snd_printk ("no post-OS interrupt.\n");2067+ dev_err(dev->card->dev, "no post-OS interrupt.\n");
2068 goto gone_bad;2068 goto gone_bad;
2069 }2069 }
2070 2070
@@ -2074,7 +2074,7 @@ wavefront_do_reset (snd_wavefront_t *dev)
2074 dev->data_port, (10*HZ));2074 dev->data_port, (10*HZ));
2075 2075
2076 if (!dev->irq_ok) {2076 if (!dev->irq_ok) {
2077- snd_printk ("no post-OS interrupt(2).\n");2077+ dev_err(dev->card->dev, "no post-OS interrupt(2).\n");
2078 goto gone_bad;2078 goto gone_bad;
2079 }2079 }
2080 2080 
@@ -2094,20 +2094,20 @@ wavefront_do_reset (snd_wavefront_t *dev)
2094 if (dev->freemem < 0)2094 if (dev->freemem < 0)
2095 goto gone_bad;2095 goto gone_bad;
2096 2096
2097- snd_printk ("available DRAM %dk\n", dev->freemem / 1024);2097+ dev_info(dev->card->dev, "available DRAM %dk\n", dev->freemem / 1024);
2098 2098 
2099 if (wavefront_write (dev, 0xf0) ||2099 if (wavefront_write (dev, 0xf0) ||
2100 wavefront_write (dev, 1) ||2100 wavefront_write (dev, 1) ||
2101 (wavefront_read (dev) < 0)) {2101 (wavefront_read (dev) < 0)) {
2102 dev->debug = 0;2102 dev->debug = 0;
2103- snd_printk ("MPU emulation mode not set.\n");2103+ dev_err(dev->card->dev, "MPU emulation mode not set.\n");
2104 goto gone_bad;2104 goto gone_bad;
2105 }2105 }
2106 2106 
2107 voices[0] = 32;2107 voices[0] = 32;
2108 2108 
2109 if (snd_wavefront_cmd (dev, WFC_SET_NVOICES, NULL, voices)) {2109 if (snd_wavefront_cmd (dev, WFC_SET_NVOICES, NULL, voices)) {
2110- snd_printk ("cannot set number of voices to 32.\n");2110+ dev_err(dev->card->dev, "cannot set number of voices to 32.\n");
2111 goto gone_bad;2111 goto gone_bad;
2112 }2112 }
2113 2113 
@@ -2187,8 +2187,8 @@ snd_wavefront_detect (snd_wavefront_card_t *card)
2187 dev->fw_version[0] = rbuf[0];2187 dev->fw_version[0] = rbuf[0];
2188 dev->fw_version[1] = rbuf[1];2188 dev->fw_version[1] = rbuf[1];
2189 2189 
2190- snd_printk ("firmware %d.%d already loaded.\n",2190+ dev_info(dev->card->dev, "firmware %d.%d already loaded.\n",
2191- rbuf[0], rbuf[1]);2191+ rbuf[0], rbuf[1]);
2192 2192 
2193 /* check that a command actually works */2193 /* check that a command actually works */
2194 2194
@@ -2197,22 +2197,24 @@ snd_wavefront_detect (snd_wavefront_card_t *card)
2197 dev->hw_version[0] = rbuf[0];2197 dev->hw_version[0] = rbuf[0];
2198 dev->hw_version[1] = rbuf[1];2198 dev->hw_version[1] = rbuf[1];
2199 } else {2199 } else {
2200- snd_printk ("not raw, but no "2200+ dev_err(dev->card->dev,
2201- "hardware version!\n");2201+ "not raw, but no hardware version!\n");
2202 return -1;2202 return -1;
2203 }2203 }
2204 2204 
2205 if (!wf_raw) {2205 if (!wf_raw) {
2206 return 0;2206 return 0;
2207 } else {2207 } else {
2208- snd_printk ("reloading firmware as you requested.\n");2208+ dev_info(dev->card->dev,
2209+ "reloading firmware as you requested.\n");
2209 dev->israw = 1;2210 dev->israw = 1;
2210 }2211 }
2211 2212 
2212 } else {2213 } else {
2213 2214 
2214 dev->israw = 1;2215 dev->israw = 1;
2215- snd_printk ("no response to firmware probe, assume raw.\n");2216+ dev_info(dev->card->dev,
2217+ "no response to firmware probe, assume raw.\n");
2216 2218 
2217 }2219 }
2218 2220 
Msound/pci/hda/cs35l41_hda.c+763-390
@@ -12,16 +12,16 @@
12#include <sound/hda_codec.h>12#include <sound/hda_codec.h>
13#include <sound/soc.h>13#include <sound/soc.h>
14#include <linux/pm_runtime.h>14#include <linux/pm_runtime.h>
15+#include <linux/spi/spi.h>
16+#include <linux/vmalloc.h>
15#include "hda_local.h"17#include "hda_local.h"
16#include "hda_auto_parser.h"18#include "hda_auto_parser.h"
17#include "hda_jack.h"19#include "hda_jack.h"
18-#include "hda_generic.h"20+#include "../generic.h"
19#include "hda_component.h"21#include "hda_component.h"
20#include "cs35l41_hda.h"22#include "cs35l41_hda.h"
21-#include "hda_cs_dsp_ctl.h"
22#include "cs35l41_hda_property.h"23#include "cs35l41_hda_property.h"
23 24 
24-#define CS35L41_FIRMWARE_ROOT "cirrus/"
25#define CS35L41_PART "cs35l41"25#define CS35L41_PART "cs35l41"
26 26 
27#define HALO_STATE_DSP_CTL_NAME "HALO_STATE"27#define HALO_STATE_DSP_CTL_NAME "HALO_STATE"
@@ -33,54 +33,97 @@
33#define CAL_AMBIENT_DSP_CTL_NAME "CAL_AMBIENT"33#define CAL_AMBIENT_DSP_CTL_NAME "CAL_AMBIENT"
34#define CAL_DSP_CTL_TYPE 534#define CAL_DSP_CTL_TYPE 5
35#define CAL_DSP_CTL_ALG 20535#define CAL_DSP_CTL_ALG 205
36+#define CS35L41_UUID "50d90cdc-3de4-4f18-b528-c7fe3b71f40d"
37+#define CS35L41_DSM_GET_MUTE 5
38+#define CS35L41_NOTIFY_EVENT 0x91
39+#define CS35L41_TUNING_SIG 0x109A4A35
40+ 
41+enum cs35l41_tuning_param_types {
42+ TUNING_PARAM_GAIN,
43+};
44+ 
45+struct cs35l41_tuning_param_hdr {
46+ __le32 tuning_index;
47+ __le32 type;
48+ __le32 size;
49+} __packed;
50+ 
51+struct cs35l41_tuning_param {
52+ struct cs35l41_tuning_param_hdr hdr;
53+ union {
54+ __le32 gain;
55+ };
56+} __packed;
57+ 
58+struct cs35l41_tuning_params {
59+ __le32 signature;
60+ __le32 version;
61+ __le32 size;
62+ __le32 num_entries;
63+ u8 data[];
64+} __packed;
65+ 
66+/* Firmware calibration controls */
67+static const struct cirrus_amp_cal_controls cs35l41_calibration_controls = {
68+ .alg_id = CAL_DSP_CTL_ALG,
69+ .mem_region = CAL_DSP_CTL_TYPE,
70+ .ambient = CAL_AMBIENT_DSP_CTL_NAME,
71+ .calr = CAL_R_DSP_CTL_NAME,
72+ .status = CAL_STATUS_DSP_CTL_NAME,
73+ .checksum = CAL_CHECKSUM_DSP_CTL_NAME,
74+};
75+ 
76+enum cs35l41_hda_fw_id {
77+ CS35L41_HDA_FW_SPK_PROT,
78+ CS35L41_HDA_FW_SPK_CALI,
79+ CS35L41_HDA_FW_SPK_DIAG,
80+ CS35L41_HDA_FW_MISC,
81+ CS35L41_HDA_NUM_FW
82+};
83+ 
84+static const char * const cs35l41_hda_fw_ids[CS35L41_HDA_NUM_FW] = {
85+ [CS35L41_HDA_FW_SPK_PROT] = "spk-prot",
86+ [CS35L41_HDA_FW_SPK_CALI] = "spk-cali",
87+ [CS35L41_HDA_FW_SPK_DIAG] = "spk-diag",
88+ [CS35L41_HDA_FW_MISC] = "misc",
89+};
36 90 
37static bool firmware_autostart = 1;91static bool firmware_autostart = 1;
38module_param(firmware_autostart, bool, 0444);92module_param(firmware_autostart, bool, 0444);
39MODULE_PARM_DESC(firmware_autostart, "Allow automatic firmware download on boot"93MODULE_PARM_DESC(firmware_autostart, "Allow automatic firmware download on boot"
40 "(0=Disable, 1=Enable) (default=1); ");94 "(0=Disable, 1=Enable) (default=1); ");
41 95 
96+static const char channel_name[3] = { 'L', 'R', 'C' };
97+ 
42static const struct reg_sequence cs35l41_hda_config[] = {98static const struct reg_sequence cs35l41_hda_config[] = {
43 { CS35L41_PLL_CLK_CTRL, 0x00000430 }, // 3072000Hz, BCLK Input, PLL_REFCLK_EN = 199 { CS35L41_PLL_CLK_CTRL, 0x00000430 }, // 3072000Hz, BCLK Input, PLL_REFCLK_EN = 1
44 { CS35L41_DSP_CLK_CTRL, 0x00000003 }, // DSP CLK EN100 { CS35L41_DSP_CLK_CTRL, 0x00000003 }, // DSP CLK EN
45 { CS35L41_GLOBAL_CLK_CTRL, 0x00000003 }, // GLOBAL_FS = 48 kHz101 { CS35L41_GLOBAL_CLK_CTRL, 0x00000003 }, // GLOBAL_FS = 48 kHz
46- { CS35L41_SP_ENABLES, 0x00010000 }, // ASP_RX1_EN = 1
47 { CS35L41_SP_RATE_CTRL, 0x00000021 }, // ASP_BCLK_FREQ = 3.072 MHz102 { CS35L41_SP_RATE_CTRL, 0x00000021 }, // ASP_BCLK_FREQ = 3.072 MHz
48 { CS35L41_SP_FORMAT, 0x20200200 }, // 32 bits RX/TX slots, I2S, clk consumer103 { CS35L41_SP_FORMAT, 0x20200200 }, // 32 bits RX/TX slots, I2S, clk consumer
49- { CS35L41_SP_HIZ_CTRL, 0x00000002 }, // Hi-Z unused
50 { CS35L41_SP_TX_WL, 0x00000018 }, // 24 cycles/slot104 { CS35L41_SP_TX_WL, 0x00000018 }, // 24 cycles/slot
51 { CS35L41_SP_RX_WL, 0x00000018 }, // 24 cycles/slot105 { CS35L41_SP_RX_WL, 0x00000018 }, // 24 cycles/slot
52- { CS35L41_DAC_PCM1_SRC, 0x00000008 }, // DACPCM1_SRC = ASPRX1
53 { CS35L41_ASP_TX1_SRC, 0x00000018 }, // ASPTX1 SRC = VMON106 { CS35L41_ASP_TX1_SRC, 0x00000018 }, // ASPTX1 SRC = VMON
54 { CS35L41_ASP_TX2_SRC, 0x00000019 }, // ASPTX2 SRC = IMON107 { CS35L41_ASP_TX2_SRC, 0x00000019 }, // ASPTX2 SRC = IMON
55- { CS35L41_ASP_TX3_SRC, 0x00000032 }, // ASPTX3 SRC = ERRVOL
56- { CS35L41_ASP_TX4_SRC, 0x00000033 }, // ASPTX4 SRC = CLASSH_TGT
57- { CS35L41_DSP1_RX1_SRC, 0x00000008 }, // DSP1RX1 SRC = ASPRX1
58- { CS35L41_DSP1_RX2_SRC, 0x00000009 }, // DSP1RX2 SRC = ASPRX2
59 { CS35L41_DSP1_RX3_SRC, 0x00000018 }, // DSP1RX3 SRC = VMON108 { CS35L41_DSP1_RX3_SRC, 0x00000018 }, // DSP1RX3 SRC = VMON
60 { CS35L41_DSP1_RX4_SRC, 0x00000019 }, // DSP1RX4 SRC = IMON109 { CS35L41_DSP1_RX4_SRC, 0x00000019 }, // DSP1RX4 SRC = IMON
110+};
111+ 
112+static const struct reg_sequence cs35l41_hda_config_no_dsp[] = {
113+ { CS35L41_SP_HIZ_CTRL, 0x00000002 }, // Hi-Z unused
114+ { CS35L41_DAC_PCM1_SRC, 0x00000008 }, // DACPCM1_SRC = ASPRX1
115+ { CS35L41_ASP_TX3_SRC, 0x00000000 }, // ASPTX3 SRC = ZERO FILL
116+ { CS35L41_ASP_TX4_SRC, 0x00000000 }, // ASPTX4 SRC = ZERO FILL
61 { CS35L41_DSP1_RX5_SRC, 0x00000020 }, // DSP1RX5 SRC = ERRVOL117 { CS35L41_DSP1_RX5_SRC, 0x00000020 }, // DSP1RX5 SRC = ERRVOL
118+ { CS35L41_DSP1_RX6_SRC, 0x00000021 }, // DSP1RX6 SRC = CLASSH_TGT
62};119};
63 120 
64static const struct reg_sequence cs35l41_hda_config_dsp[] = {121static const struct reg_sequence cs35l41_hda_config_dsp[] = {
65- { CS35L41_PLL_CLK_CTRL, 0x00000430 }, // 3072000Hz, BCLK Input, PLL_REFCLK_EN = 1
66- { CS35L41_DSP_CLK_CTRL, 0x00000003 }, // DSP CLK EN
67- { CS35L41_GLOBAL_CLK_CTRL, 0x00000003 }, // GLOBAL_FS = 48 kHz
68- { CS35L41_SP_ENABLES, 0x00010001 }, // ASP_RX1_EN = 1, ASP_TX1_EN = 1
69- { CS35L41_SP_RATE_CTRL, 0x00000021 }, // ASP_BCLK_FREQ = 3.072 MHz
70- { CS35L41_SP_FORMAT, 0x20200200 }, // 32 bits RX/TX slots, I2S, clk consumer
71 { CS35L41_SP_HIZ_CTRL, 0x00000003 }, // Hi-Z unused/disabled122 { CS35L41_SP_HIZ_CTRL, 0x00000003 }, // Hi-Z unused/disabled
72- { CS35L41_SP_TX_WL, 0x00000018 }, // 24 cycles/slot123+ { CS35L41_DAC_PCM1_SRC, 0x00000032 }, // DACPCM1_SRC = DSP1TX1
73- { CS35L41_SP_RX_WL, 0x00000018 }, // 24 cycles/slot
74- { CS35L41_DAC_PCM1_SRC, 0x00000032 }, // DACPCM1_SRC = ERR_VOL
75- { CS35L41_ASP_TX1_SRC, 0x00000018 }, // ASPTX1 SRC = VMON
76- { CS35L41_ASP_TX2_SRC, 0x00000019 }, // ASPTX2 SRC = IMON
77 { CS35L41_ASP_TX3_SRC, 0x00000028 }, // ASPTX3 SRC = VPMON124 { CS35L41_ASP_TX3_SRC, 0x00000028 }, // ASPTX3 SRC = VPMON
78 { CS35L41_ASP_TX4_SRC, 0x00000029 }, // ASPTX4 SRC = VBSTMON125 { CS35L41_ASP_TX4_SRC, 0x00000029 }, // ASPTX4 SRC = VBSTMON
79- { CS35L41_DSP1_RX1_SRC, 0x00000008 }, // DSP1RX1 SRC = ASPRX1126+ { CS35L41_DSP1_RX6_SRC, 0x00000029 }, // DSP1RX6 SRC = VBSTMON
80- { CS35L41_DSP1_RX2_SRC, 0x00000008 }, // DSP1RX2 SRC = ASPRX1
81- { CS35L41_DSP1_RX3_SRC, 0x00000018 }, // DSP1RX3 SRC = VMON
82- { CS35L41_DSP1_RX4_SRC, 0x00000019 }, // DSP1RX4 SRC = IMON
83- { CS35L41_DSP1_RX5_SRC, 0x00000029 }, // DSP1RX5 SRC = VBSTMON
84};127};
85 128 
86static const struct reg_sequence cs35l41_hda_unmute[] = {129static const struct reg_sequence cs35l41_hda_unmute[] = {
@@ -88,34 +131,39 @@ static const struct reg_sequence cs35l41_hda_unmute[] = {
88 { CS35L41_AMP_GAIN_CTRL, 0x00000084 }, // AMP_GAIN_PCM 4.5 dB131 { CS35L41_AMP_GAIN_CTRL, 0x00000084 }, // AMP_GAIN_PCM 4.5 dB
89};132};
90 133 
91-static const struct reg_sequence cs35l41_hda_unmute_dsp[] = {
92- { CS35L41_AMP_DIG_VOL_CTRL, 0x00008000 }, // AMP_HPF_PCM_EN = 1, AMP_VOL_PCM 0.0 dB
93- { CS35L41_AMP_GAIN_CTRL, 0x00000233 }, // AMP_GAIN_PCM = 17.5dB AMP_GAIN_PDM = 19.5dB
94-};
95- 
96static const struct reg_sequence cs35l41_hda_mute[] = {134static const struct reg_sequence cs35l41_hda_mute[] = {
97 { CS35L41_AMP_GAIN_CTRL, 0x00000000 }, // AMP_GAIN_PCM 0.5 dB135 { CS35L41_AMP_GAIN_CTRL, 0x00000000 }, // AMP_GAIN_PCM 0.5 dB
98 { CS35L41_AMP_DIG_VOL_CTRL, 0x0000A678 }, // AMP_HPF_PCM_EN = 1, AMP_VOL_PCM Mute136 { CS35L41_AMP_DIG_VOL_CTRL, 0x0000A678 }, // AMP_HPF_PCM_EN = 1, AMP_VOL_PCM Mute
99};137};
100 138 
101-static void cs35l41_add_controls(struct cs35l41_hda *cs35l41)
102-{
103- struct hda_cs_dsp_ctl_info info;
104- 
105- info.device_name = cs35l41->amp_name;
106- info.fw_type = cs35l41->firmware_type;
107- info.card = cs35l41->codec->card;
108- 
109- hda_cs_dsp_add_controls(&cs35l41->cs_dsp, &info);
110-}
111- 
112static const struct cs_dsp_client_ops client_ops = {139static const struct cs_dsp_client_ops client_ops = {
113- .control_remove = hda_cs_dsp_control_remove,140+ /* cs_dsp requires the client to provide this even if it is empty */
114};141};
115 142 
143+static int cs35l41_request_tuning_param_file(struct cs35l41_hda *cs35l41, char *tuning_filename,
144+ const struct firmware **firmware, char **filename,
145+ const char *ssid)
146+{
147+ int ret = 0;
148+ 
149+ /* Filename is the same as the tuning file with "cfg" suffix */
150+ *filename = kasprintf(GFP_KERNEL, "%scfg", tuning_filename);
151+ if (*filename == NULL)
152+ return -ENOMEM;
153+ 
154+ ret = firmware_request_nowarn(firmware, *filename, cs35l41->dev);
155+ if (ret != 0) {
156+ dev_dbg(cs35l41->dev, "Failed to request '%s'\n", *filename);
157+ kfree(*filename);
158+ *filename = NULL;
159+ }
160+ 
161+ return ret;
162+}
163+ 
116static int cs35l41_request_firmware_file(struct cs35l41_hda *cs35l41,164static int cs35l41_request_firmware_file(struct cs35l41_hda *cs35l41,
117 const struct firmware **firmware, char **filename,165 const struct firmware **firmware, char **filename,
118- const char *dir, const char *ssid, const char *amp_name,166+ const char *ssid, const char *amp_name,
119 int spkid, const char *filetype)167 int spkid, const char *filetype)
120{168{
121 const char * const dsp_name = cs35l41->cs_dsp.name;169 const char * const dsp_name = cs35l41->cs_dsp.name;
@@ -123,24 +171,24 @@ static int cs35l41_request_firmware_file(struct cs35l41_hda *cs35l41,
123 int ret = 0;171 int ret = 0;
124 172 
125 if (spkid > -1 && ssid && amp_name)173 if (spkid > -1 && ssid && amp_name)
126- *filename = kasprintf(GFP_KERNEL, "%s%s-%s-%s-%s-spkid%d-%s.%s", dir, CS35L41_PART,174+ *filename = kasprintf(GFP_KERNEL, "cirrus/%s-%s-%s-%s-spkid%d-%s.%s", CS35L41_PART,
127- dsp_name, hda_cs_dsp_fw_ids[cs35l41->firmware_type],175+ dsp_name, cs35l41_hda_fw_ids[cs35l41->firmware_type],
128 ssid, spkid, amp_name, filetype);176 ssid, spkid, amp_name, filetype);
129 else if (spkid > -1 && ssid)177 else if (spkid > -1 && ssid)
130- *filename = kasprintf(GFP_KERNEL, "%s%s-%s-%s-%s-spkid%d.%s", dir, CS35L41_PART,178+ *filename = kasprintf(GFP_KERNEL, "cirrus/%s-%s-%s-%s-spkid%d.%s", CS35L41_PART,
131- dsp_name, hda_cs_dsp_fw_ids[cs35l41->firmware_type],179+ dsp_name, cs35l41_hda_fw_ids[cs35l41->firmware_type],
132 ssid, spkid, filetype);180 ssid, spkid, filetype);
133 else if (ssid && amp_name)181 else if (ssid && amp_name)
134- *filename = kasprintf(GFP_KERNEL, "%s%s-%s-%s-%s-%s.%s", dir, CS35L41_PART,182+ *filename = kasprintf(GFP_KERNEL, "cirrus/%s-%s-%s-%s-%s.%s", CS35L41_PART,
135- dsp_name, hda_cs_dsp_fw_ids[cs35l41->firmware_type],183+ dsp_name, cs35l41_hda_fw_ids[cs35l41->firmware_type],
136 ssid, amp_name, filetype);184 ssid, amp_name, filetype);
137 else if (ssid)185 else if (ssid)
138- *filename = kasprintf(GFP_KERNEL, "%s%s-%s-%s-%s.%s", dir, CS35L41_PART,186+ *filename = kasprintf(GFP_KERNEL, "cirrus/%s-%s-%s-%s.%s", CS35L41_PART,
139- dsp_name, hda_cs_dsp_fw_ids[cs35l41->firmware_type],187+ dsp_name, cs35l41_hda_fw_ids[cs35l41->firmware_type],
140 ssid, filetype);188 ssid, filetype);
141 else189 else
142- *filename = kasprintf(GFP_KERNEL, "%s%s-%s-%s.%s", dir, CS35L41_PART,190+ *filename = kasprintf(GFP_KERNEL, "cirrus/%s-%s-%s.%s", CS35L41_PART,
143- dsp_name, hda_cs_dsp_fw_ids[cs35l41->firmware_type],191+ dsp_name, cs35l41_hda_fw_ids[cs35l41->firmware_type],
144 filetype);192 filetype);
145 193 
146 if (*filename == NULL)194 if (*filename == NULL)
@@ -180,13 +228,11 @@ static int cs35l41_request_firmware_files_spkid(struct cs35l41_hda *cs35l41,
180 228 
181 /* try cirrus/part-dspN-fwtype-sub<-spkidN><-ampname>.wmfw */229 /* try cirrus/part-dspN-fwtype-sub<-spkidN><-ampname>.wmfw */
182 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,230 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,
183- CS35L41_FIRMWARE_ROOT,
184 cs35l41->acpi_subsystem_id, cs35l41->amp_name,231 cs35l41->acpi_subsystem_id, cs35l41->amp_name,
185 cs35l41->speaker_id, "wmfw");232 cs35l41->speaker_id, "wmfw");
186 if (!ret) {233 if (!ret) {
187 /* try cirrus/part-dspN-fwtype-sub<-spkidN><-ampname>.bin */234 /* try cirrus/part-dspN-fwtype-sub<-spkidN><-ampname>.bin */
188 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,235 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,
189- CS35L41_FIRMWARE_ROOT,
190 cs35l41->acpi_subsystem_id, cs35l41->amp_name,236 cs35l41->acpi_subsystem_id, cs35l41->amp_name,
191 cs35l41->speaker_id, "bin");237 cs35l41->speaker_id, "bin");
192 if (ret)238 if (ret)
@@ -197,12 +243,11 @@ static int cs35l41_request_firmware_files_spkid(struct cs35l41_hda *cs35l41,
197 243 
198 /* try cirrus/part-dspN-fwtype-sub<-ampname>.wmfw */244 /* try cirrus/part-dspN-fwtype-sub<-ampname>.wmfw */
199 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,245 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,
200- CS35L41_FIRMWARE_ROOT, cs35l41->acpi_subsystem_id,246+ cs35l41->acpi_subsystem_id,
201 cs35l41->amp_name, -1, "wmfw");247 cs35l41->amp_name, -1, "wmfw");
202 if (!ret) {248 if (!ret) {
203 /* try cirrus/part-dspN-fwtype-sub<-spkidN><-ampname>.bin */249 /* try cirrus/part-dspN-fwtype-sub<-spkidN><-ampname>.bin */
204 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,250 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,
205- CS35L41_FIRMWARE_ROOT,
206 cs35l41->acpi_subsystem_id, cs35l41->amp_name,251 cs35l41->acpi_subsystem_id, cs35l41->amp_name,
207 cs35l41->speaker_id, "bin");252 cs35l41->speaker_id, "bin");
208 if (ret)253 if (ret)
@@ -213,18 +258,17 @@ static int cs35l41_request_firmware_files_spkid(struct cs35l41_hda *cs35l41,
213 258 
214 /* try cirrus/part-dspN-fwtype-sub<-spkidN>.wmfw */259 /* try cirrus/part-dspN-fwtype-sub<-spkidN>.wmfw */
215 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,260 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,
216- CS35L41_FIRMWARE_ROOT, cs35l41->acpi_subsystem_id,261+ cs35l41->acpi_subsystem_id,
217 NULL, cs35l41->speaker_id, "wmfw");262 NULL, cs35l41->speaker_id, "wmfw");
218 if (!ret) {263 if (!ret) {
219 /* try cirrus/part-dspN-fwtype-sub<-spkidN><-ampname>.bin */264 /* try cirrus/part-dspN-fwtype-sub<-spkidN><-ampname>.bin */
220 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,265 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,
221- CS35L41_FIRMWARE_ROOT,
222 cs35l41->acpi_subsystem_id,266 cs35l41->acpi_subsystem_id,
223 cs35l41->amp_name, cs35l41->speaker_id, "bin");267 cs35l41->amp_name, cs35l41->speaker_id, "bin");
224 if (ret)268 if (ret)
225 /* try cirrus/part-dspN-fwtype-sub<-spkidN>.bin */269 /* try cirrus/part-dspN-fwtype-sub<-spkidN>.bin */
226 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware,270 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware,
227- coeff_filename, CS35L41_FIRMWARE_ROOT,271+ coeff_filename,
228 cs35l41->acpi_subsystem_id, NULL,272 cs35l41->acpi_subsystem_id, NULL,
229 cs35l41->speaker_id, "bin");273 cs35l41->speaker_id, "bin");
230 if (ret)274 if (ret)
@@ -235,18 +279,17 @@ static int cs35l41_request_firmware_files_spkid(struct cs35l41_hda *cs35l41,
235 279 
236 /* try cirrus/part-dspN-fwtype-sub.wmfw */280 /* try cirrus/part-dspN-fwtype-sub.wmfw */
237 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,281 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,
238- CS35L41_FIRMWARE_ROOT, cs35l41->acpi_subsystem_id,282+ cs35l41->acpi_subsystem_id,
239 NULL, -1, "wmfw");283 NULL, -1, "wmfw");
240 if (!ret) {284 if (!ret) {
241 /* try cirrus/part-dspN-fwtype-sub<-spkidN><-ampname>.bin */285 /* try cirrus/part-dspN-fwtype-sub<-spkidN><-ampname>.bin */
242 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,286 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,
243- CS35L41_FIRMWARE_ROOT,
244 cs35l41->acpi_subsystem_id, cs35l41->amp_name,287 cs35l41->acpi_subsystem_id, cs35l41->amp_name,
245 cs35l41->speaker_id, "bin");288 cs35l41->speaker_id, "bin");
246 if (ret)289 if (ret)
247 /* try cirrus/part-dspN-fwtype-sub<-spkidN>.bin */290 /* try cirrus/part-dspN-fwtype-sub<-spkidN>.bin */
248 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware,291 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware,
249- coeff_filename, CS35L41_FIRMWARE_ROOT,292+ coeff_filename,
250 cs35l41->acpi_subsystem_id, NULL,293 cs35l41->acpi_subsystem_id, NULL,
251 cs35l41->speaker_id, "bin");294 cs35l41->speaker_id, "bin");
252 if (ret)295 if (ret)
@@ -273,13 +316,13 @@ static int cs35l41_fallback_firmware_file(struct cs35l41_hda *cs35l41,
273 316 
274 /* fallback try cirrus/part-dspN-fwtype.wmfw */317 /* fallback try cirrus/part-dspN-fwtype.wmfw */
275 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,318 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,
276- CS35L41_FIRMWARE_ROOT, NULL, NULL, -1, "wmfw");319+ NULL, NULL, -1, "wmfw");
277 if (ret)320 if (ret)
278 goto err;321 goto err;
279 322 
280 /* fallback try cirrus/part-dspN-fwtype.bin */323 /* fallback try cirrus/part-dspN-fwtype.bin */
281 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,324 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,
282- CS35L41_FIRMWARE_ROOT, NULL, NULL, -1, "bin");325+ NULL, NULL, -1, "bin");
283 if (ret) {326 if (ret) {
284 release_firmware(*wmfw_firmware);327 release_firmware(*wmfw_firmware);
285 kfree(*wmfw_filename);328 kfree(*wmfw_filename);
@@ -308,12 +351,11 @@ static int cs35l41_request_firmware_files(struct cs35l41_hda *cs35l41,
308 351 
309 /* try cirrus/part-dspN-fwtype-sub<-ampname>.wmfw */352 /* try cirrus/part-dspN-fwtype-sub<-ampname>.wmfw */
310 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,353 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,
311- CS35L41_FIRMWARE_ROOT, cs35l41->acpi_subsystem_id,354+ cs35l41->acpi_subsystem_id,
312 cs35l41->amp_name, -1, "wmfw");355 cs35l41->amp_name, -1, "wmfw");
313 if (!ret) {356 if (!ret) {
314 /* try cirrus/part-dspN-fwtype-sub<-ampname>.bin */357 /* try cirrus/part-dspN-fwtype-sub<-ampname>.bin */
315 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,358 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,
316- CS35L41_FIRMWARE_ROOT,
317 cs35l41->acpi_subsystem_id, cs35l41->amp_name,359 cs35l41->acpi_subsystem_id, cs35l41->amp_name,
318 -1, "bin");360 -1, "bin");
319 if (ret)361 if (ret)
@@ -324,18 +366,16 @@ static int cs35l41_request_firmware_files(struct cs35l41_hda *cs35l41,
324 366 
325 /* try cirrus/part-dspN-fwtype-sub.wmfw */367 /* try cirrus/part-dspN-fwtype-sub.wmfw */
326 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,368 ret = cs35l41_request_firmware_file(cs35l41, wmfw_firmware, wmfw_filename,
327- CS35L41_FIRMWARE_ROOT, cs35l41->acpi_subsystem_id,369+ cs35l41->acpi_subsystem_id,
328 NULL, -1, "wmfw");370 NULL, -1, "wmfw");
329 if (!ret) {371 if (!ret) {
330 /* try cirrus/part-dspN-fwtype-sub<-ampname>.bin */372 /* try cirrus/part-dspN-fwtype-sub<-ampname>.bin */
331 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,373 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,
332- CS35L41_FIRMWARE_ROOT,
333 cs35l41->acpi_subsystem_id,374 cs35l41->acpi_subsystem_id,
334 cs35l41->amp_name, -1, "bin");375 cs35l41->amp_name, -1, "bin");
335 if (ret)376 if (ret)
336 /* try cirrus/part-dspN-fwtype-sub.bin */377 /* try cirrus/part-dspN-fwtype-sub.bin */
337 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,378 ret = cs35l41_request_firmware_file(cs35l41, coeff_firmware, coeff_filename,
338- CS35L41_FIRMWARE_ROOT,
339 cs35l41->acpi_subsystem_id, NULL, -1,379 cs35l41->acpi_subsystem_id, NULL, -1,
340 "bin");380 "bin");
341 if (ret)381 if (ret)
@@ -357,95 +397,162 @@ static int cs35l41_request_firmware_files(struct cs35l41_hda *cs35l41,
357 coeff_firmware, coeff_filename);397 coeff_firmware, coeff_filename);
358}398}
359 399 
360-#if IS_ENABLED(CONFIG_EFI)400+ 
361-static int cs35l41_apply_calibration(struct cs35l41_hda *cs35l41, __be32 ambient, __be32 r0,401+static void cs35l41_hda_apply_calibration(struct cs35l41_hda *cs35l41)
362- __be32 status, __be32 checksum)
363{402{
364 int ret;403 int ret;
365 404 
366- ret = hda_cs_dsp_write_ctl(&cs35l41->cs_dsp, CAL_AMBIENT_DSP_CTL_NAME, CAL_DSP_CTL_TYPE,405+ if (!cs35l41->cal_data_valid)
367- CAL_DSP_CTL_ALG, &ambient, 4);406+ return;
407+ 
408+ ret = cs_amp_write_cal_coeffs(&cs35l41->cs_dsp, &cs35l41_calibration_controls,
409+ &cs35l41->cal_data);
410+ if (ret < 0)
411+ dev_warn(cs35l41->dev, "Failed to apply calibration: %d\n", ret);
412+ else
413+ dev_info(cs35l41->dev, "Calibration applied: R0=%d\n", cs35l41->cal_data.calR);
414+}
415+ 
416+static int cs35l41_read_silicon_uid(struct cs35l41_hda *cs35l41, u64 *uid)
417+{
418+ u32 tmp;
419+ int ret;
420+ 
421+ ret = regmap_read(cs35l41->regmap, CS35L41_DIE_STS2, &tmp);
368 if (ret) {422 if (ret) {
369- dev_err(cs35l41->dev, "Cannot Write Control: %s - %d\n", CAL_AMBIENT_DSP_CTL_NAME,423+ dev_err(cs35l41->dev, "Cannot obtain CS35L41_DIE_STS2: %d\n", ret);
370- ret);
371 return ret;424 return ret;
372 }425 }
373- ret = hda_cs_dsp_write_ctl(&cs35l41->cs_dsp, CAL_R_DSP_CTL_NAME, CAL_DSP_CTL_TYPE,426+ 
374- CAL_DSP_CTL_ALG, &r0, 4);427+ *uid = tmp;
428+ *uid <<= 32;
429+ 
430+ ret = regmap_read(cs35l41->regmap, CS35L41_DIE_STS1, &tmp);
375 if (ret) {431 if (ret) {
376- dev_err(cs35l41->dev, "Cannot Write Control: %s - %d\n", CAL_R_DSP_CTL_NAME, ret);432+ dev_err(cs35l41->dev, "Cannot obtain CS35L41_DIE_STS1: %d\n", ret);
377 return ret;433 return ret;
378 }434 }
379- ret = hda_cs_dsp_write_ctl(&cs35l41->cs_dsp, CAL_STATUS_DSP_CTL_NAME, CAL_DSP_CTL_TYPE,435+ 
380- CAL_DSP_CTL_ALG, &status, 4);436+ *uid |= tmp;
381- if (ret) {437+ 
382- dev_err(cs35l41->dev, "Cannot Write Control: %s - %d\n", CAL_STATUS_DSP_CTL_NAME,438+ dev_dbg(cs35l41->dev, "UniqueID = %#llx\n", *uid);
383- ret);439+ 
440+ return 0;
441+}
442+ 
443+static int cs35l41_get_calibration(struct cs35l41_hda *cs35l41)
444+{
445+ u64 silicon_uid;
446+ int ret;
447+ 
448+ ret = cs35l41_read_silicon_uid(cs35l41, &silicon_uid);
449+ if (ret < 0)
384 return ret;450 return ret;
451+ 
452+ ret = cs_amp_get_efi_calibration_data(cs35l41->dev, silicon_uid,
453+ cs35l41->index,
454+ &cs35l41->cal_data);
455+ 
456+ /* Only return an error status if probe should be aborted */
457+ if ((ret == -ENOENT) || (ret == -EOVERFLOW))
458+ return 0;
459+ 
460+ if (ret < 0)
461+ return ret;
462+ 
463+ cs35l41->cal_data_valid = true;
464+ 
465+ return 0;
466+}
467+ 
468+ 
469+static void cs35l41_set_default_tuning_params(struct cs35l41_hda *cs35l41)
470+{
471+ cs35l41->tuning_gain = DEFAULT_AMP_GAIN_PCM;
472+}
473+ 
474+static int cs35l41_read_tuning_params(struct cs35l41_hda *cs35l41, const struct firmware *firmware)
475+{
476+ struct cs35l41_tuning_params *params;
477+ unsigned int offset = 0;
478+ unsigned int end;
479+ int i;
480+ 
481+ params = (void *)&firmware->data[0];
482+ 
483+ if (le32_to_cpu(params->size) != firmware->size) {
484+ dev_err(cs35l41->dev, "Wrong Size for Tuning Param file. Expected %d got %zu\n",
485+ le32_to_cpu(params->size), firmware->size);
486+ return -EINVAL;
385 }487 }
386- ret = hda_cs_dsp_write_ctl(&cs35l41->cs_dsp, CAL_CHECKSUM_DSP_CTL_NAME, CAL_DSP_CTL_TYPE,488+ 
387- CAL_DSP_CTL_ALG, &checksum, 4);489+ if (le32_to_cpu(params->version) != 1) {
388- if (ret) {490+ dev_err(cs35l41->dev, "Unsupported Tuning Param Version: %d\n",
389- dev_err(cs35l41->dev, "Cannot Write Control: %s - %d\n", CAL_CHECKSUM_DSP_CTL_NAME,491+ le32_to_cpu(params->version));
390- ret);492+ return -EINVAL;
391- return ret;493+ }
494+ 
495+ if (le32_to_cpu(params->signature) != CS35L41_TUNING_SIG) {
496+ dev_err(cs35l41->dev,
497+ "Mismatched Signature for Tuning Param file. Expected %#x got %#x\n",
498+ CS35L41_TUNING_SIG, le32_to_cpu(params->signature));
499+ return -EINVAL;
500+ }
501+ 
502+ end = firmware->size - sizeof(struct cs35l41_tuning_params);
503+ 
504+ for (i = 0; i < le32_to_cpu(params->num_entries); i++) {
505+ struct cs35l41_tuning_param *param;
506+ 
507+ if ((offset >= end) || ((offset + sizeof(struct cs35l41_tuning_param_hdr)) >= end))
508+ return -EFAULT;
509+ 
510+ param = (void *)&params->data[offset];
511+ offset += le32_to_cpu(param->hdr.size);
512+ 
513+ if (offset > end)
514+ return -EFAULT;
515+ 
516+ switch (le32_to_cpu(param->hdr.type)) {
517+ case TUNING_PARAM_GAIN:
518+ cs35l41->tuning_gain = le32_to_cpu(param->gain);
519+ dev_dbg(cs35l41->dev, "Applying Gain: %d\n", cs35l41->tuning_gain);
520+ break;
521+ default:
522+ break;
523+ }
392 }524 }
393 525 
394 return 0;526 return 0;
395}527}
396 528 
397-static int cs35l41_save_calibration(struct cs35l41_hda *cs35l41)529+static int cs35l41_load_tuning_params(struct cs35l41_hda *cs35l41, char *tuning_filename)
398{530{
399- static efi_guid_t efi_guid = EFI_GUID(0x02f9af02, 0x7734, 0x4233, 0xb4, 0x3d, 0x93, 0xfe,531+ const struct firmware *tuning_param_file = NULL;
400- 0x5a, 0xa3, 0x5d, 0xb3);532+ char *tuning_param_filename = NULL;
401- static efi_char16_t efi_name[] = L"CirrusSmartAmpCalibrationData";533+ int ret;
402- const struct cs35l41_amp_efi_data *efi_data;
403- const struct cs35l41_amp_cal_data *cl;
404- unsigned long data_size = 0;
405- efi_status_t status;
406- int ret = 0;
407- u8 *data = NULL;
408- u32 attr;
409 534 
410- /* Get real size of UEFI variable */535+ ret = cs35l41_request_tuning_param_file(cs35l41, tuning_filename, &tuning_param_file,
411- status = efi.get_variable(efi_name, &efi_guid, &attr, &data_size, data);536+ &tuning_param_filename, cs35l41->acpi_subsystem_id);
412- if (status == EFI_BUFFER_TOO_SMALL) {537+ if (ret) {
413- ret = -ENODEV;538+ dev_dbg(cs35l41->dev, "Missing Tuning Param for file: %s: %d\n", tuning_filename,
414- /* Allocate data buffer of data_size bytes */539+ ret);
415- data = vmalloc(data_size);540+ return 0;
416- if (!data)
417- return -ENOMEM;
418- /* Get variable contents into buffer */
419- status = efi.get_variable(efi_name, &efi_guid, &attr, &data_size, data);
420- if (status == EFI_SUCCESS) {
421- efi_data = (struct cs35l41_amp_efi_data *)data;
422- dev_dbg(cs35l41->dev, "Calibration: Size=%d, Amp Count=%d\n",
423- efi_data->size, efi_data->count);
424- if (efi_data->count > cs35l41->index) {
425- cl = &efi_data->data[cs35l41->index];
426- dev_dbg(cs35l41->dev,
427- "Calibration: Ambient=%02x, Status=%02x, R0=%d\n",
428- cl->calAmbient, cl->calStatus, cl->calR);
429- 
430- /* Calibration can only be applied whilst the DSP is not running */
431- ret = cs35l41_apply_calibration(cs35l41,
432- cpu_to_be32(cl->calAmbient),
433- cpu_to_be32(cl->calR),
434- cpu_to_be32(cl->calStatus),
435- cpu_to_be32(cl->calR + 1));
436- }
437- }
438- vfree(data);
439 }541 }
542+ 
543+ ret = cs35l41_read_tuning_params(cs35l41, tuning_param_file);
544+ if (ret) {
545+ dev_err(cs35l41->dev, "Error reading Tuning Params from file: %s: %d\n",
546+ tuning_param_filename, ret);
547+ /* Reset to default Tuning Parameters */
548+ cs35l41_set_default_tuning_params(cs35l41);
549+ }
550+ 
551+ release_firmware(tuning_param_file);
552+ kfree(tuning_param_filename);
553+ 
440 return ret;554 return ret;
441}555}
442-#else
443-static int cs35l41_save_calibration(struct cs35l41_hda *cs35l41)
444-{
445- dev_warn(cs35l41->dev, "Calibration not supported without EFI support.\n");
446- return 0;
447-}
448-#endif
449 556 
450static int cs35l41_init_dsp(struct cs35l41_hda *cs35l41)557static int cs35l41_init_dsp(struct cs35l41_hda *cs35l41)
451{558{
@@ -466,27 +573,33 @@ static int cs35l41_init_dsp(struct cs35l41_hda *cs35l41)
466 cs35l41->halo_initialized = true;573 cs35l41->halo_initialized = true;
467 }574 }
468 575 
576+ cs35l41_set_default_tuning_params(cs35l41);
577+ 
469 ret = cs35l41_request_firmware_files(cs35l41, &wmfw_firmware, &wmfw_filename,578 ret = cs35l41_request_firmware_files(cs35l41, &wmfw_firmware, &wmfw_filename,
470 &coeff_firmware, &coeff_filename);579 &coeff_firmware, &coeff_filename);
471 if (ret < 0)580 if (ret < 0)
472 return ret;581 return ret;
473 582 
474 dev_dbg(cs35l41->dev, "Loading WMFW Firmware: %s\n", wmfw_filename);583 dev_dbg(cs35l41->dev, "Loading WMFW Firmware: %s\n", wmfw_filename);
475- if (coeff_filename)584+ if (coeff_filename) {
476 dev_dbg(cs35l41->dev, "Loading Coefficient File: %s\n", coeff_filename);585 dev_dbg(cs35l41->dev, "Loading Coefficient File: %s\n", coeff_filename);
477- else586+ ret = cs35l41_load_tuning_params(cs35l41, coeff_filename);
587+ if (ret)
588+ dev_warn(cs35l41->dev, "Unable to load Tuning Parameters: %d\n", ret);
589+ } else {
478 dev_warn(cs35l41->dev, "No Coefficient File available.\n");590 dev_warn(cs35l41->dev, "No Coefficient File available.\n");
591+ }
479 592 
480 ret = cs_dsp_power_up(dsp, wmfw_firmware, wmfw_filename, coeff_firmware, coeff_filename,593 ret = cs_dsp_power_up(dsp, wmfw_firmware, wmfw_filename, coeff_firmware, coeff_filename,
481- hda_cs_dsp_fw_ids[cs35l41->firmware_type]);594+ cs35l41_hda_fw_ids[cs35l41->firmware_type]);
482 if (ret)595 if (ret)
483- goto err_release;596+ goto err;
484 597 
485- cs35l41_add_controls(cs35l41);598+ cs35l41_hda_apply_calibration(cs35l41);
486 599 
487- ret = cs35l41_save_calibration(cs35l41);600+err:
488- 601+ if (ret)
489-err_release:602+ cs35l41_set_default_tuning_params(cs35l41);
490 release_firmware(wmfw_firmware);603 release_firmware(wmfw_firmware);
491 release_firmware(coeff_firmware);604 release_firmware(coeff_firmware);
492 kfree(wmfw_filename);605 kfree(wmfw_filename);
@@ -499,9 +612,9 @@ static void cs35l41_shutdown_dsp(struct cs35l41_hda *cs35l41)
499{612{
500 struct cs_dsp *dsp = &cs35l41->cs_dsp;613 struct cs_dsp *dsp = &cs35l41->cs_dsp;
501 614 
615+ cs35l41_set_default_tuning_params(cs35l41);
502 cs_dsp_stop(dsp);616 cs_dsp_stop(dsp);
503 cs_dsp_power_down(dsp);617 cs_dsp_power_down(dsp);
504- cs35l41->firmware_running = false;
505 dev_dbg(cs35l41->dev, "Unloaded Firmware\n");618 dev_dbg(cs35l41->dev, "Unloaded Firmware\n");
506}619}
507 620 
@@ -511,11 +624,10 @@ static void cs35l41_remove_dsp(struct cs35l41_hda *cs35l41)
511 624 
512 cancel_work_sync(&cs35l41->fw_load_work);625 cancel_work_sync(&cs35l41->fw_load_work);
513 626 
514- mutex_lock(&cs35l41->fw_mutex);627+ guard(mutex)(&cs35l41->fw_mutex);
515 cs35l41_shutdown_dsp(cs35l41);628 cs35l41_shutdown_dsp(cs35l41);
516 cs_dsp_remove(dsp);629 cs_dsp_remove(dsp);
517 cs35l41->halo_initialized = false;630 cs35l41->halo_initialized = false;
518- mutex_unlock(&cs35l41->fw_mutex);
519}631}
520 632 
521/* Protection release cycle to get the speaker out of Safe-Mode */633/* Protection release cycle to get the speaker out of Safe-Mode */
@@ -533,6 +645,41 @@ static void cs35l41_irq_release(struct cs35l41_hda *cs35l41)
533 cs35l41->irq_errors = 0;645 cs35l41->irq_errors = 0;
534}646}
535 647 
648+static void cs35l41_update_mixer(struct cs35l41_hda *cs35l41)
649+{
650+ struct regmap *reg = cs35l41->regmap;
651+ unsigned int asp_en = 0;
652+ unsigned int dsp1rx2_src = 0;
653+ 
654+ regmap_multi_reg_write(reg, cs35l41_hda_config, ARRAY_SIZE(cs35l41_hda_config));
655+ 
656+ if (cs35l41->cs_dsp.running) {
657+ asp_en |= CS35L41_ASP_TX1_EN_MASK; // ASP_TX1_EN = 1
658+ regmap_multi_reg_write(reg, cs35l41_hda_config_dsp,
659+ ARRAY_SIZE(cs35l41_hda_config_dsp));
660+ if (cs35l41->hw_cfg.bst_type == CS35L41_INT_BOOST)
661+ regmap_write(reg, CS35L41_DSP1_RX5_SRC, CS35L41_INPUT_SRC_VPMON);
662+ else
663+ regmap_write(reg, CS35L41_DSP1_RX5_SRC, CS35L41_INPUT_SRC_VBSTMON);
664+ } else {
665+ regmap_multi_reg_write(reg, cs35l41_hda_config_no_dsp,
666+ ARRAY_SIZE(cs35l41_hda_config_no_dsp));
667+ }
668+ 
669+ if (cs35l41->hw_cfg.spk_pos == CS35L41_CENTER) {
670+ asp_en |= CS35L41_ASP_RX2_EN_MASK; // ASP_RX2_EN = 1
671+ dsp1rx2_src = 0x00000009; // DSP1RX2 SRC = ASPRX2
672+ } else {
673+ dsp1rx2_src = 0x00000008; // DSP1RX2 SRC = ASPRX1
674+ }
675+ 
676+ asp_en |= CS35L41_ASP_RX1_EN_MASK; // ASP_RX1_EN = 1
677+ 
678+ regmap_write(reg, CS35L41_SP_ENABLES, asp_en);
679+ regmap_write(reg, CS35L41_DSP1_RX1_SRC, 0x00000008); // DSP1RX1 SRC = ASPRX1
680+ regmap_write(reg, CS35L41_DSP1_RX2_SRC, dsp1rx2_src);
681+}
682+ 
536static void cs35l41_hda_play_start(struct device *dev)683static void cs35l41_hda_play_start(struct device *dev)
537{684{
538 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);685 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
@@ -547,15 +694,13 @@ static void cs35l41_hda_play_start(struct device *dev)
547 694 
548 cs35l41->playback_started = true;695 cs35l41->playback_started = true;
549 696 
550- if (cs35l41->firmware_running) {697+ cs35l41_update_mixer(cs35l41);
551- regmap_multi_reg_write(reg, cs35l41_hda_config_dsp,698+ 
552- ARRAY_SIZE(cs35l41_hda_config_dsp));699+ if (cs35l41->cs_dsp.running) {
553 regmap_update_bits(reg, CS35L41_PWR_CTRL2,700 regmap_update_bits(reg, CS35L41_PWR_CTRL2,
554 CS35L41_VMON_EN_MASK | CS35L41_IMON_EN_MASK,701 CS35L41_VMON_EN_MASK | CS35L41_IMON_EN_MASK,
555 1 << CS35L41_VMON_EN_SHIFT | 1 << CS35L41_IMON_EN_SHIFT);702 1 << CS35L41_VMON_EN_SHIFT | 1 << CS35L41_IMON_EN_SHIFT);
556 cs35l41_set_cspl_mbox_cmd(cs35l41->dev, reg, CSPL_MBOX_CMD_RESUME);703 cs35l41_set_cspl_mbox_cmd(cs35l41->dev, reg, CSPL_MBOX_CMD_RESUME);
557- } else {
558- regmap_multi_reg_write(reg, cs35l41_hda_config, ARRAY_SIZE(cs35l41_hda_config));
559 }704 }
560 regmap_update_bits(reg, CS35L41_PWR_CTRL2, CS35L41_AMP_EN_MASK, 1 << CS35L41_AMP_EN_SHIFT);705 regmap_update_bits(reg, CS35L41_PWR_CTRL2, CS35L41_AMP_EN_MASK, 1 << CS35L41_AMP_EN_SHIFT);
561 if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST)706 if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST)
@@ -563,6 +708,37 @@ static void cs35l41_hda_play_start(struct device *dev)
563 708 
564}709}
565 710 
711+static void cs35l41_mute(struct device *dev, bool mute)
712+{
713+ struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
714+ struct regmap *reg = cs35l41->regmap;
715+ unsigned int amp_gain;
716+ 
717+ dev_dbg(dev, "Mute(%d:%d) Playback Started: %d\n", mute, cs35l41->mute_override,
718+ cs35l41->playback_started);
719+ 
720+ if (cs35l41->playback_started) {
721+ if (mute || cs35l41->mute_override) {
722+ dev_dbg(dev, "Muting\n");
723+ regmap_multi_reg_write(reg, cs35l41_hda_mute, ARRAY_SIZE(cs35l41_hda_mute));
724+ } else {
725+ dev_dbg(dev, "Unmuting\n");
726+ if (cs35l41->cs_dsp.running) {
727+ dev_dbg(dev, "Using Tuned Gain: %d\n", cs35l41->tuning_gain);
728+ amp_gain = (cs35l41->tuning_gain << CS35L41_AMP_GAIN_PCM_SHIFT) |
729+ (DEFAULT_AMP_GAIN_PDM << CS35L41_AMP_GAIN_PDM_SHIFT);
730+ 
731+ /* AMP_HPF_PCM_EN = 1, AMP_VOL_PCM 0.0 dB */
732+ regmap_write(reg, CS35L41_AMP_DIG_VOL_CTRL, 0x00008000);
733+ regmap_write(reg, CS35L41_AMP_GAIN_CTRL, amp_gain);
734+ } else {
735+ regmap_multi_reg_write(reg, cs35l41_hda_unmute,
736+ ARRAY_SIZE(cs35l41_hda_unmute));
737+ }
738+ }
739+ }
740+}
741+ 
566static void cs35l41_hda_play_done(struct device *dev)742static void cs35l41_hda_play_done(struct device *dev)
567{743{
568 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);744 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
@@ -571,14 +747,8 @@ static void cs35l41_hda_play_done(struct device *dev)
571 dev_dbg(dev, "Play (Complete)\n");747 dev_dbg(dev, "Play (Complete)\n");
572 748 
573 cs35l41_global_enable(dev, reg, cs35l41->hw_cfg.bst_type, 1,749 cs35l41_global_enable(dev, reg, cs35l41->hw_cfg.bst_type, 1,
574- cs35l41->firmware_running);750+ &cs35l41->cs_dsp);
575- if (cs35l41->firmware_running) {751+ cs35l41_mute(dev, false);
576- regmap_multi_reg_write(reg, cs35l41_hda_unmute_dsp,
577- ARRAY_SIZE(cs35l41_hda_unmute_dsp));
578- } else {
579- regmap_multi_reg_write(reg, cs35l41_hda_unmute,
580- ARRAY_SIZE(cs35l41_hda_unmute));
581- }
582}752}
583 753 
584static void cs35l41_hda_pause_start(struct device *dev)754static void cs35l41_hda_pause_start(struct device *dev)
@@ -588,9 +758,9 @@ static void cs35l41_hda_pause_start(struct device *dev)
588 758 
589 dev_dbg(dev, "Pause (Start)\n");759 dev_dbg(dev, "Pause (Start)\n");
590 760 
591- regmap_multi_reg_write(reg, cs35l41_hda_mute, ARRAY_SIZE(cs35l41_hda_mute));761+ cs35l41_mute(dev, true);
592 cs35l41_global_enable(dev, reg, cs35l41->hw_cfg.bst_type, 0,762 cs35l41_global_enable(dev, reg, cs35l41->hw_cfg.bst_type, 0,
593- cs35l41->firmware_running);763+ &cs35l41->cs_dsp);
594}764}
595 765 
596static void cs35l41_hda_pause_done(struct device *dev)766static void cs35l41_hda_pause_done(struct device *dev)
@@ -603,7 +773,7 @@ static void cs35l41_hda_pause_done(struct device *dev)
603 regmap_update_bits(reg, CS35L41_PWR_CTRL2, CS35L41_AMP_EN_MASK, 0 << CS35L41_AMP_EN_SHIFT);773 regmap_update_bits(reg, CS35L41_PWR_CTRL2, CS35L41_AMP_EN_MASK, 0 << CS35L41_AMP_EN_SHIFT);
604 if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST)774 if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST)
605 regmap_write(reg, CS35L41_GPIO1_CTRL1, 0x00000001);775 regmap_write(reg, CS35L41_GPIO1_CTRL1, 0x00000001);
606- if (cs35l41->firmware_running) {776+ if (cs35l41->cs_dsp.running) {
607 cs35l41_set_cspl_mbox_cmd(dev, reg, CSPL_MBOX_CMD_PAUSE);777 cs35l41_set_cspl_mbox_cmd(dev, reg, CSPL_MBOX_CMD_PAUSE);
608 regmap_update_bits(reg, CS35L41_PWR_CTRL2,778 regmap_update_bits(reg, CS35L41_PWR_CTRL2,
609 CS35L41_VMON_EN_MASK | CS35L41_IMON_EN_MASK,779 CS35L41_VMON_EN_MASK | CS35L41_IMON_EN_MASK,
@@ -619,9 +789,9 @@ static void cs35l41_hda_pre_playback_hook(struct device *dev, int action)
619 789 
620 switch (action) {790 switch (action) {
621 case HDA_GEN_PCM_ACT_CLEANUP:791 case HDA_GEN_PCM_ACT_CLEANUP:
622- mutex_lock(&cs35l41->fw_mutex);792+ scoped_guard(mutex, &cs35l41->fw_mutex) {
623- cs35l41_hda_pause_start(dev);793+ cs35l41_hda_pause_start(dev);
624- mutex_unlock(&cs35l41->fw_mutex);794+ }
625 break;795 break;
626 default:796 default:
627 break;797 break;
@@ -642,30 +812,29 @@ static void cs35l41_hda_playback_hook(struct device *dev, int action)
642 pm_runtime_get_sync(dev);812 pm_runtime_get_sync(dev);
643 break;813 break;
644 case HDA_GEN_PCM_ACT_PREPARE:814 case HDA_GEN_PCM_ACT_PREPARE:
645- mutex_lock(&cs35l41->fw_mutex);815+ scoped_guard(mutex, &cs35l41->fw_mutex) {
646- cs35l41_hda_play_start(dev);816+ cs35l41_hda_play_start(dev);
647- mutex_unlock(&cs35l41->fw_mutex);817+ }
648 break;818 break;
649 case HDA_GEN_PCM_ACT_CLEANUP:819 case HDA_GEN_PCM_ACT_CLEANUP:
650- mutex_lock(&cs35l41->fw_mutex);820+ scoped_guard(mutex, &cs35l41->fw_mutex) {
651- cs35l41_hda_pause_done(dev);821+ cs35l41_hda_pause_done(dev);
652- mutex_unlock(&cs35l41->fw_mutex);822+ }
653 break;823 break;
654 case HDA_GEN_PCM_ACT_CLOSE:824 case HDA_GEN_PCM_ACT_CLOSE:
655- mutex_lock(&cs35l41->fw_mutex);825+ scoped_guard(mutex, &cs35l41->fw_mutex) {
656- if (!cs35l41->firmware_running && cs35l41->request_fw_load &&826+ if (!cs35l41->cs_dsp.running && cs35l41->request_fw_load &&
657- !cs35l41->fw_request_ongoing) {827+ !cs35l41->fw_request_ongoing) {
658- dev_info(dev, "Requesting Firmware Load after HDA_GEN_PCM_ACT_CLOSE\n");828+ dev_info(dev, "Requesting Firmware Load after HDA_GEN_PCM_ACT_CLOSE\n");
659- cs35l41->fw_request_ongoing = true;829+ cs35l41->fw_request_ongoing = true;
660- schedule_work(&cs35l41->fw_load_work);830+ schedule_work(&cs35l41->fw_load_work);
831+ }
661 }832 }
662- mutex_unlock(&cs35l41->fw_mutex);
663 833 
664 /*834 /*
665 * Playback must be finished for all amps before we start runtime suspend.835 * Playback must be finished for all amps before we start runtime suspend.
666 * This ensures no amps are playing back when we start putting them to sleep.836 * This ensures no amps are playing back when we start putting them to sleep.
667 */837 */
668- pm_runtime_mark_last_busy(dev);
669 pm_runtime_put_autosuspend(dev);838 pm_runtime_put_autosuspend(dev);
670 break;839 break;
671 default:840 default:
@@ -679,72 +848,82 @@ static void cs35l41_hda_post_playback_hook(struct device *dev, int action)
679 848 
680 switch (action) {849 switch (action) {
681 case HDA_GEN_PCM_ACT_PREPARE:850 case HDA_GEN_PCM_ACT_PREPARE:
682- mutex_lock(&cs35l41->fw_mutex);851+ scoped_guard(mutex, &cs35l41->fw_mutex) {
683- cs35l41_hda_play_done(dev);852+ cs35l41_hda_play_done(dev);
684- mutex_unlock(&cs35l41->fw_mutex);853+ }
685 break;854 break;
686 default:855 default:
687 break;856 break;
688 }857 }
689}858}
690 859 
691-static int cs35l41_hda_channel_map(struct device *dev, unsigned int tx_num, unsigned int *tx_slot,860+static int cs35l41_hda_channel_map(struct cs35l41_hda *cs35l41)
692- unsigned int rx_num, unsigned int *rx_slot)
693{861{
694- struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);862+ unsigned int tx_num = 0;
695- static const char * const channel_name[] = { "L", "R" };863+ unsigned int *tx_slot = NULL;
864+ unsigned int rx_num;
865+ unsigned int *rx_slot;
866+ unsigned int mono = 0;
696 867 
697 if (!cs35l41->amp_name) {868 if (!cs35l41->amp_name) {
698- if (*rx_slot >= ARRAY_SIZE(channel_name))869+ if (cs35l41->hw_cfg.spk_pos >= ARRAY_SIZE(channel_name))
699 return -EINVAL;870 return -EINVAL;
700 871 
701- cs35l41->amp_name = devm_kasprintf(cs35l41->dev, GFP_KERNEL, "%s%d",872+ cs35l41->amp_name = devm_kasprintf(cs35l41->dev, GFP_KERNEL, "%c%d",
702- channel_name[*rx_slot], cs35l41->channel_index);873+ channel_name[cs35l41->hw_cfg.spk_pos],
874+ cs35l41->channel_index);
703 if (!cs35l41->amp_name)875 if (!cs35l41->amp_name)
704 return -ENOMEM;876 return -ENOMEM;
705 }877 }
706 878 
879+ rx_num = 1;
880+ if (cs35l41->hw_cfg.spk_pos == CS35L41_CENTER)
881+ rx_slot = &mono;
882+ else
883+ rx_slot = &cs35l41->hw_cfg.spk_pos;
884+ 
707 return cs35l41_set_channels(cs35l41->dev, cs35l41->regmap, tx_num, tx_slot, rx_num,885 return cs35l41_set_channels(cs35l41->dev, cs35l41->regmap, tx_num, tx_slot, rx_num,
708 rx_slot);886 rx_slot);
709}887}
710 888 
889+static int cs35l41_verify_id(struct cs35l41_hda *cs35l41, unsigned int *regid, unsigned int *reg_revid)
890+{
891+ unsigned int mtl_revid, chipid;
892+ int ret;
893+ 
894+ ret = regmap_read(cs35l41->regmap, CS35L41_DEVID, regid);
895+ if (ret) {
896+ dev_err_probe(cs35l41->dev, ret, "Get Device ID failed\n");
897+ return ret;
898+ }
899+ 
900+ ret = regmap_read(cs35l41->regmap, CS35L41_REVID, reg_revid);
901+ if (ret) {
902+ dev_err_probe(cs35l41->dev, ret, "Get Revision ID failed\n");
903+ return ret;
904+ }
905+ 
906+ mtl_revid = *reg_revid & CS35L41_MTLREVID_MASK;
907+ 
908+ chipid = (mtl_revid % 2) ? CS35L41R_CHIP_ID : CS35L41_CHIP_ID;
909+ if (*regid != chipid) {
910+ dev_err(cs35l41->dev, "CS35L41 Device ID (%X). Expected ID %X\n", *regid, chipid);
911+ return -ENODEV;
912+ }
913+ 
914+ return 0;
915+}
916+ 
711static int cs35l41_ready_for_reset(struct cs35l41_hda *cs35l41)917static int cs35l41_ready_for_reset(struct cs35l41_hda *cs35l41)
712{918{
713- int ret = 0;919+ guard(mutex)(&cs35l41->fw_mutex);
714- 920+ if (cs35l41->cs_dsp.running) {
715- mutex_lock(&cs35l41->fw_mutex);921+ cs35l41->cs_dsp.running = false;
716- if (cs35l41->firmware_running) {922+ cs35l41->cs_dsp.booted = false;
717- 
718- regcache_cache_only(cs35l41->regmap, false);
719- 
720- ret = cs35l41_exit_hibernate(cs35l41->dev, cs35l41->regmap);
721- if (ret) {
722- dev_warn(cs35l41->dev, "Unable to exit Hibernate.");
723- goto err;
724- }
725- 
726- /* Test key needs to be unlocked to allow the OTP settings to re-apply */
727- cs35l41_test_key_unlock(cs35l41->dev, cs35l41->regmap);
728- ret = regcache_sync(cs35l41->regmap);
729- cs35l41_test_key_lock(cs35l41->dev, cs35l41->regmap);
730- if (ret) {
731- dev_err(cs35l41->dev, "Failed to restore register cache: %d\n", ret);
732- goto err;
733- }
734- 
735- if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST)
736- cs35l41_init_boost(cs35l41->dev, cs35l41->regmap, &cs35l41->hw_cfg);
737- 
738- cs35l41_shutdown_dsp(cs35l41);
739- cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type);
740 }923 }
741-err:
742- regcache_cache_only(cs35l41->regmap, true);
743 regcache_mark_dirty(cs35l41->regmap);924 regcache_mark_dirty(cs35l41->regmap);
744 925 
745- mutex_unlock(&cs35l41->fw_mutex);926+ return 0;
746- 
747- return ret;
748}927}
749 928 
750static int cs35l41_system_suspend_prep(struct device *dev)929static int cs35l41_system_suspend_prep(struct device *dev)
@@ -758,10 +937,9 @@ static int cs35l41_system_suspend_prep(struct device *dev)
758 return 0; /* don't block the whole system suspend */937 return 0; /* don't block the whole system suspend */
759 }938 }
760 939 
761- mutex_lock(&cs35l41->fw_mutex);940+ guard(mutex)(&cs35l41->fw_mutex);
762 if (cs35l41->playback_started)941 if (cs35l41->playback_started)
763 cs35l41_hda_pause_start(dev);942 cs35l41_hda_pause_start(dev);
764- mutex_unlock(&cs35l41->fw_mutex);
765 943 
766 return 0;944 return 0;
767}945}
@@ -778,10 +956,10 @@ static int cs35l41_system_suspend(struct device *dev)
778 return 0; /* don't block the whole system suspend */956 return 0; /* don't block the whole system suspend */
779 }957 }
780 958 
781- mutex_lock(&cs35l41->fw_mutex);959+ scoped_guard(mutex, &cs35l41->fw_mutex) {
782- if (cs35l41->playback_started)960+ if (cs35l41->playback_started)
783- cs35l41_hda_pause_done(dev);961+ cs35l41_hda_pause_done(dev);
784- mutex_unlock(&cs35l41->fw_mutex);962+ }
785 963 
786 ret = pm_runtime_force_suspend(dev);964 ret = pm_runtime_force_suspend(dev);
787 if (ret) {965 if (ret) {
@@ -791,17 +969,44 @@ static int cs35l41_system_suspend(struct device *dev)
791 969 
792 /* Shutdown DSP before system suspend */970 /* Shutdown DSP before system suspend */
793 ret = cs35l41_ready_for_reset(cs35l41);971 ret = cs35l41_ready_for_reset(cs35l41);
794- 
795 if (ret)972 if (ret)
796 dev_err(dev, "System Suspend Failed, not ready for Reset: %d\n", ret);973 dev_err(dev, "System Suspend Failed, not ready for Reset: %d\n", ret);
797 974 
798- /*975+ if (cs35l41->reset_gpio) {
799- * Reset GPIO may be shared, so cannot reset here.976+ dev_info(cs35l41->dev, "Asserting Reset\n");
800- * However beyond this point, amps may be powered down.977+ gpiod_set_value_cansleep(cs35l41->reset_gpio, 0);
801- */978+ usleep_range(2000, 2100);
979+ }
980+ 
981+ dev_dbg(cs35l41->dev, "System Suspended\n");
982+ 
802 return ret;983 return ret;
803}984}
804 985 
986+static int cs35l41_wait_boot_done(struct cs35l41_hda *cs35l41)
987+{
988+ unsigned int int_status;
989+ int ret;
990+ 
991+ ret = regmap_read_poll_timeout(cs35l41->regmap, CS35L41_IRQ1_STATUS4, int_status,
992+ int_status & CS35L41_OTP_BOOT_DONE, 1000, 100000);
993+ if (ret) {
994+ dev_err(cs35l41->dev, "Failed waiting for OTP_BOOT_DONE\n");
995+ return ret;
996+ }
997+ 
998+ ret = regmap_read(cs35l41->regmap, CS35L41_IRQ1_STATUS3, &int_status);
999+ if (ret || (int_status & CS35L41_OTP_BOOT_ERR)) {
1000+ dev_err(cs35l41->dev, "OTP Boot status %x error\n",
1001+ int_status & CS35L41_OTP_BOOT_ERR);
1002+ if (!ret)
1003+ ret = -EIO;
1004+ return ret;
1005+ }
1006+ 
1007+ return 0;
1008+}
1009+ 
805static int cs35l41_system_resume(struct device *dev)1010static int cs35l41_system_resume(struct device *dev)
806{1011{
807 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);1012 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
@@ -815,25 +1020,36 @@ static int cs35l41_system_resume(struct device *dev)
815 }1020 }
816 1021 
817 if (cs35l41->reset_gpio) {1022 if (cs35l41->reset_gpio) {
1023+ gpiod_set_value_cansleep(cs35l41->reset_gpio, 0);
818 usleep_range(2000, 2100);1024 usleep_range(2000, 2100);
819 gpiod_set_value_cansleep(cs35l41->reset_gpio, 1);1025 gpiod_set_value_cansleep(cs35l41->reset_gpio, 1);
820 }1026 }
821 1027 
822 usleep_range(2000, 2100);1028 usleep_range(2000, 2100);
823 1029 
1030+ regcache_cache_only(cs35l41->regmap, false);
1031+ 
1032+ regmap_write(cs35l41->regmap, CS35L41_SFT_RESET, CS35L41_SOFTWARE_RESET);
1033+ usleep_range(2000, 2100);
1034+ 
1035+ ret = cs35l41_wait_boot_done(cs35l41);
1036+ if (ret)
1037+ return ret;
1038+ 
1039+ regcache_cache_only(cs35l41->regmap, true);
1040+ 
824 ret = pm_runtime_force_resume(dev);1041 ret = pm_runtime_force_resume(dev);
825 if (ret) {1042 if (ret) {
826 dev_err(dev, "System Resume Failed: Unable to runtime resume: %d\n", ret);1043 dev_err(dev, "System Resume Failed: Unable to runtime resume: %d\n", ret);
827 return ret;1044 return ret;
828 }1045 }
829 1046 
830- mutex_lock(&cs35l41->fw_mutex);1047+ guard(mutex)(&cs35l41->fw_mutex);
831 1048 
832 if (cs35l41->request_fw_load && !cs35l41->fw_request_ongoing) {1049 if (cs35l41->request_fw_load && !cs35l41->fw_request_ongoing) {
833 cs35l41->fw_request_ongoing = true;1050 cs35l41->fw_request_ongoing = true;
834 schedule_work(&cs35l41->fw_load_work);1051 schedule_work(&cs35l41->fw_load_work);
835 }1052 }
836- mutex_unlock(&cs35l41->fw_mutex);
837 1053 
838 return ret;1054 return ret;
839}1055}
@@ -850,7 +1066,7 @@ static int cs35l41_runtime_idle(struct device *dev)
850static int cs35l41_runtime_suspend(struct device *dev)1066static int cs35l41_runtime_suspend(struct device *dev)
851{1067{
852 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);1068 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
853- int ret = 0;1069+ int ret;
854 1070 
855 dev_dbg(cs35l41->dev, "Runtime Suspend\n");1071 dev_dbg(cs35l41->dev, "Runtime Suspend\n");
856 1072 
@@ -859,13 +1075,13 @@ static int cs35l41_runtime_suspend(struct device *dev)
859 return 0;1075 return 0;
860 }1076 }
861 1077 
862- mutex_lock(&cs35l41->fw_mutex);1078+ guard(mutex)(&cs35l41->fw_mutex);
863 1079 
864- if (cs35l41->firmware_running) {1080+ if (cs35l41->cs_dsp.running) {
865 ret = cs35l41_enter_hibernate(cs35l41->dev, cs35l41->regmap,1081 ret = cs35l41_enter_hibernate(cs35l41->dev, cs35l41->regmap,
866 cs35l41->hw_cfg.bst_type);1082 cs35l41->hw_cfg.bst_type);
867 if (ret)1083 if (ret)
868- goto err;1084+ return ret;
869 } else {1085 } else {
870 cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type);1086 cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type);
871 }1087 }
@@ -873,16 +1089,14 @@ static int cs35l41_runtime_suspend(struct device *dev)
873 regcache_cache_only(cs35l41->regmap, true);1089 regcache_cache_only(cs35l41->regmap, true);
874 regcache_mark_dirty(cs35l41->regmap);1090 regcache_mark_dirty(cs35l41->regmap);
875 1091 
876-err:1092+ return 0;
877- mutex_unlock(&cs35l41->fw_mutex);
878- 
879- return ret;
880}1093}
881 1094 
882static int cs35l41_runtime_resume(struct device *dev)1095static int cs35l41_runtime_resume(struct device *dev)
883{1096{
884 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);1097 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
885- int ret = 0;1098+ unsigned int regid, reg_revid;
1099+ int ret;
886 1100 
887 dev_dbg(cs35l41->dev, "Runtime Resume\n");1101 dev_dbg(cs35l41->dev, "Runtime Resume\n");
888 1102 
@@ -891,41 +1105,57 @@ static int cs35l41_runtime_resume(struct device *dev)
891 return 0;1105 return 0;
892 }1106 }
893 1107 
894- mutex_lock(&cs35l41->fw_mutex);1108+ guard(mutex)(&cs35l41->fw_mutex);
895 1109 
896 regcache_cache_only(cs35l41->regmap, false);1110 regcache_cache_only(cs35l41->regmap, false);
897 1111 
898- if (cs35l41->firmware_running) {1112+ if (cs35l41->cs_dsp.running) {
899 ret = cs35l41_exit_hibernate(cs35l41->dev, cs35l41->regmap);1113 ret = cs35l41_exit_hibernate(cs35l41->dev, cs35l41->regmap);
900 if (ret) {1114 if (ret) {
901 dev_warn(cs35l41->dev, "Unable to exit Hibernate.");1115 dev_warn(cs35l41->dev, "Unable to exit Hibernate.");
902- goto err;1116+ return ret;
903 }1117 }
904 }1118 }
905 1119 
1120+ ret = cs35l41_verify_id(cs35l41, &regid, &reg_revid);
1121+ if (ret)
1122+ return ret;
1123+ 
906 /* Test key needs to be unlocked to allow the OTP settings to re-apply */1124 /* Test key needs to be unlocked to allow the OTP settings to re-apply */
907 cs35l41_test_key_unlock(cs35l41->dev, cs35l41->regmap);1125 cs35l41_test_key_unlock(cs35l41->dev, cs35l41->regmap);
908 ret = regcache_sync(cs35l41->regmap);1126 ret = regcache_sync(cs35l41->regmap);
909 cs35l41_test_key_lock(cs35l41->dev, cs35l41->regmap);1127 cs35l41_test_key_lock(cs35l41->dev, cs35l41->regmap);
910 if (ret) {1128 if (ret) {
911 dev_err(cs35l41->dev, "Failed to restore register cache: %d\n", ret);1129 dev_err(cs35l41->dev, "Failed to restore register cache: %d\n", ret);
912- goto err;1130+ return ret;
913 }1131 }
914 1132 
915 if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST)1133 if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST)
916 cs35l41_init_boost(cs35l41->dev, cs35l41->regmap, &cs35l41->hw_cfg);1134 cs35l41_init_boost(cs35l41->dev, cs35l41->regmap, &cs35l41->hw_cfg);
917 1135 
918-err:1136+ dev_dbg(cs35l41->dev, "CS35L41 Resumed (%x), Revision: %02X\n", regid, reg_revid);
919- mutex_unlock(&cs35l41->fw_mutex);
920 1137 
921- return ret;1138+ return 0;
1139+}
1140+ 
1141+static int cs35l41_hda_read_ctl(struct cs_dsp *dsp, const char *name, int type,
1142+ unsigned int alg, void *buf, size_t len)
1143+{
1144+ guard(mutex)(&dsp->pwr_lock);
1145+ return cs_dsp_coeff_read_ctrl(cs_dsp_get_ctl(dsp, name, type, alg), 0, buf, len);
922}1146}
923 1147 
924static int cs35l41_smart_amp(struct cs35l41_hda *cs35l41)1148static int cs35l41_smart_amp(struct cs35l41_hda *cs35l41)
925{1149{
1150+ unsigned int fw_status;
926 __be32 halo_sts;1151 __be32 halo_sts;
927 int ret;1152 int ret;
928 1153 
1154+ if (cs35l41->bypass_fw) {
1155+ dev_warn(cs35l41->dev, "Bypassing Firmware.\n");
1156+ return 0;
1157+ }
1158+ 
929 ret = cs35l41_init_dsp(cs35l41);1159 ret = cs35l41_init_dsp(cs35l41);
930 if (ret) {1160 if (ret) {
931 dev_warn(cs35l41->dev, "Cannot Initialize Firmware. Error: %d\n", ret);1161 dev_warn(cs35l41->dev, "Cannot Initialize Firmware. Error: %d\n", ret);
@@ -944,7 +1174,7 @@ static int cs35l41_smart_amp(struct cs35l41_hda *cs35l41)
944 goto clean_dsp;1174 goto clean_dsp;
945 }1175 }
946 1176 
947- ret = read_poll_timeout(hda_cs_dsp_read_ctl, ret,1177+ ret = read_poll_timeout(cs35l41_hda_read_ctl, ret,
948 be32_to_cpu(halo_sts) == HALO_STATE_CODE_RUN,1178 be32_to_cpu(halo_sts) == HALO_STATE_CODE_RUN,
949 1000, 15000, false, &cs35l41->cs_dsp, HALO_STATE_DSP_CTL_NAME,1179 1000, 15000, false, &cs35l41->cs_dsp, HALO_STATE_DSP_CTL_NAME,
950 HALO_STATE_DSP_CTL_TYPE, HALO_STATE_DSP_CTL_ALG,1180 HALO_STATE_DSP_CTL_TYPE, HALO_STATE_DSP_CTL_ALG,
@@ -956,13 +1186,32 @@ static int cs35l41_smart_amp(struct cs35l41_hda *cs35l41)
956 goto clean_dsp;1186 goto clean_dsp;
957 }1187 }
958 1188 
1189+ ret = regmap_read(cs35l41->regmap, CS35L41_DSP_MBOX_2, &fw_status);
1190+ if (ret < 0) {
1191+ dev_err(cs35l41->dev,
1192+ "Failed to read firmware status: %d\n", ret);
1193+ goto clean_dsp;
1194+ }
1195+ 
1196+ switch (fw_status) {
1197+ case CSPL_MBOX_STS_RUNNING:
1198+ case CSPL_MBOX_STS_PAUSED:
1199+ break;
1200+ default:
1201+ dev_err(cs35l41->dev, "Firmware status is invalid: %u\n",
1202+ fw_status);
1203+ ret = -EINVAL;
1204+ goto clean_dsp;
1205+ }
1206+ 
959 ret = cs35l41_set_cspl_mbox_cmd(cs35l41->dev, cs35l41->regmap, CSPL_MBOX_CMD_PAUSE);1207 ret = cs35l41_set_cspl_mbox_cmd(cs35l41->dev, cs35l41->regmap, CSPL_MBOX_CMD_PAUSE);
960 if (ret) {1208 if (ret) {
961 dev_err(cs35l41->dev, "Error waiting for DSP to pause: %u\n", ret);1209 dev_err(cs35l41->dev, "Error waiting for DSP to pause: %u\n", ret);
962 goto clean_dsp;1210 goto clean_dsp;
963 }1211 }
964 1212 
965- cs35l41->firmware_running = true;1213+ dev_info(cs35l41->dev, "Firmware Loaded - Type: %s, Gain: %d\n",
1214+ cs35l41_hda_fw_ids[cs35l41->firmware_type], cs35l41->tuning_gain);
966 1215 
967 return 0;1216 return 0;
968 1217 
@@ -973,10 +1222,10 @@ static int cs35l41_smart_amp(struct cs35l41_hda *cs35l41)
973 1222 
974static void cs35l41_load_firmware(struct cs35l41_hda *cs35l41, bool load)1223static void cs35l41_load_firmware(struct cs35l41_hda *cs35l41, bool load)
975{1224{
976- if (cs35l41->firmware_running && !load) {1225+ if (cs35l41->cs_dsp.running && !load) {
977 dev_dbg(cs35l41->dev, "Unloading Firmware\n");1226 dev_dbg(cs35l41->dev, "Unloading Firmware\n");
978 cs35l41_shutdown_dsp(cs35l41);1227 cs35l41_shutdown_dsp(cs35l41);
979- } else if (!cs35l41->firmware_running && load) {1228+ } else if (!cs35l41->cs_dsp.running && load) {
980 dev_dbg(cs35l41->dev, "Loading Firmware\n");1229 dev_dbg(cs35l41->dev, "Loading Firmware\n");
981 cs35l41_smart_amp(cs35l41);1230 cs35l41_smart_amp(cs35l41);
982 } else {1231 } else {
@@ -993,24 +1242,31 @@ static int cs35l41_fw_load_ctl_get(struct snd_kcontrol *kcontrol,
993 return 0;1242 return 0;
994}1243}
995 1244 
1245+static int cs35l41_mute_override_ctl_get(struct snd_kcontrol *kcontrol,
1246+ struct snd_ctl_elem_value *ucontrol)
1247+{
1248+ struct cs35l41_hda *cs35l41 = snd_kcontrol_chip(kcontrol);
1249+ 
1250+ ucontrol->value.integer.value[0] = cs35l41->mute_override;
1251+ return 0;
1252+}
1253+ 
996static void cs35l41_fw_load_work(struct work_struct *work)1254static void cs35l41_fw_load_work(struct work_struct *work)
997{1255{
998 struct cs35l41_hda *cs35l41 = container_of(work, struct cs35l41_hda, fw_load_work);1256 struct cs35l41_hda *cs35l41 = container_of(work, struct cs35l41_hda, fw_load_work);
999 1257 
1000 pm_runtime_get_sync(cs35l41->dev);1258 pm_runtime_get_sync(cs35l41->dev);
1001 1259 
1002- mutex_lock(&cs35l41->fw_mutex);1260+ scoped_guard(mutex, &cs35l41->fw_mutex) {
1261+ /* Recheck if playback is ongoing, mutex will block playback during firmware loading */
1262+ if (cs35l41->playback_started)
1263+ dev_err(cs35l41->dev, "Cannot Load/Unload firmware during Playback. Retrying...\n");
1264+ else
1265+ cs35l41_load_firmware(cs35l41, cs35l41->request_fw_load);
1003 1266 
1004- /* Recheck if playback is ongoing, mutex will block playback during firmware loading */1267+ cs35l41->fw_request_ongoing = false;
1005- if (cs35l41->playback_started)1268+ }
1006- dev_err(cs35l41->dev, "Cannot Load/Unload firmware during Playback. Retrying...\n");
1007- else
1008- cs35l41_load_firmware(cs35l41, cs35l41->request_fw_load);
1009 1269 
1010- cs35l41->fw_request_ongoing = false;
1011- mutex_unlock(&cs35l41->fw_mutex);
1012- 
1013- pm_runtime_mark_last_busy(cs35l41->dev);
1014 pm_runtime_put_autosuspend(cs35l41->dev);1270 pm_runtime_put_autosuspend(cs35l41->dev);
1015}1271}
1016 1272 
@@ -1055,7 +1311,7 @@ static int cs35l41_fw_type_ctl_put(struct snd_kcontrol *kcontrol,
1055{1311{
1056 struct cs35l41_hda *cs35l41 = snd_kcontrol_chip(kcontrol);1312 struct cs35l41_hda *cs35l41 = snd_kcontrol_chip(kcontrol);
1057 1313 
1058- if (ucontrol->value.enumerated.item[0] < HDA_CS_DSP_NUM_FW) {1314+ if (ucontrol->value.enumerated.item[0] < CS35L41_HDA_NUM_FW) {
1059 if (cs35l41->firmware_type != ucontrol->value.enumerated.item[0]) {1315 if (cs35l41->firmware_type != ucontrol->value.enumerated.item[0]) {
1060 cs35l41->firmware_type = ucontrol->value.enumerated.item[0];1316 cs35l41->firmware_type = ucontrol->value.enumerated.item[0];
1061 return 1;1317 return 1;
@@ -1069,13 +1325,14 @@ static int cs35l41_fw_type_ctl_put(struct snd_kcontrol *kcontrol,
1069 1325 
1070static int cs35l41_fw_type_ctl_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo)1326static int cs35l41_fw_type_ctl_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo)
1071{1327{
1072- return snd_ctl_enum_info(uinfo, 1, ARRAY_SIZE(hda_cs_dsp_fw_ids), hda_cs_dsp_fw_ids);1328+ return snd_ctl_enum_info(uinfo, 1, ARRAY_SIZE(cs35l41_hda_fw_ids), cs35l41_hda_fw_ids);
1073}1329}
1074 1330 
1075static int cs35l41_create_controls(struct cs35l41_hda *cs35l41)1331static int cs35l41_create_controls(struct cs35l41_hda *cs35l41)
1076{1332{
1077 char fw_type_ctl_name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN];1333 char fw_type_ctl_name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN];
1078 char fw_load_ctl_name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN];1334 char fw_load_ctl_name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN];
1335+ char mute_override_ctl_name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN];
1079 struct snd_kcontrol_new fw_type_ctl = {1336 struct snd_kcontrol_new fw_type_ctl = {
1080 .name = fw_type_ctl_name,1337 .name = fw_type_ctl_name,
1081 .iface = SNDRV_CTL_ELEM_IFACE_CARD,1338 .iface = SNDRV_CTL_ELEM_IFACE_CARD,
@@ -1090,12 +1347,21 @@ static int cs35l41_create_controls(struct cs35l41_hda *cs35l41)
1090 .get = cs35l41_fw_load_ctl_get,1347 .get = cs35l41_fw_load_ctl_get,
1091 .put = cs35l41_fw_load_ctl_put,1348 .put = cs35l41_fw_load_ctl_put,
1092 };1349 };
1350+ struct snd_kcontrol_new mute_override_ctl = {
1351+ .name = mute_override_ctl_name,
1352+ .iface = SNDRV_CTL_ELEM_IFACE_CARD,
1353+ .info = snd_ctl_boolean_mono_info,
1354+ .access = SNDRV_CTL_ELEM_ACCESS_READ | SNDRV_CTL_ELEM_ACCESS_VOLATILE,
1355+ .get = cs35l41_mute_override_ctl_get,
1356+ };
1093 int ret;1357 int ret;
1094 1358 
1095 scnprintf(fw_type_ctl_name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, "%s DSP1 Firmware Type",1359 scnprintf(fw_type_ctl_name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, "%s DSP1 Firmware Type",
1096 cs35l41->amp_name);1360 cs35l41->amp_name);
1097 scnprintf(fw_load_ctl_name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, "%s DSP1 Firmware Load",1361 scnprintf(fw_load_ctl_name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, "%s DSP1 Firmware Load",
1098 cs35l41->amp_name);1362 cs35l41->amp_name);
1363+ scnprintf(mute_override_ctl_name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, "%s Forced Mute Status",
1364+ cs35l41->amp_name);
1099 1365 
1100 ret = snd_ctl_add(cs35l41->codec->card, snd_ctl_new1(&fw_type_ctl, cs35l41));1366 ret = snd_ctl_add(cs35l41->codec->card, snd_ctl_new1(&fw_type_ctl, cs35l41));
1101 if (ret) {1367 if (ret) {
@@ -1113,35 +1379,95 @@ static int cs35l41_create_controls(struct cs35l41_hda *cs35l41)
1113 1379 
1114 dev_dbg(cs35l41->dev, "Added Control %s\n", fw_load_ctl.name);1380 dev_dbg(cs35l41->dev, "Added Control %s\n", fw_load_ctl.name);
1115 1381 
1382+ ret = snd_ctl_add(cs35l41->codec->card, snd_ctl_new1(&mute_override_ctl, cs35l41));
1383+ if (ret) {
1384+ dev_err(cs35l41->dev, "Failed to add KControl %s = %d\n", mute_override_ctl.name,
1385+ ret);
1386+ return ret;
1387+ }
1388+ 
1389+ dev_dbg(cs35l41->dev, "Added Control %s\n", mute_override_ctl.name);
1390+ 
1116 return 0;1391 return 0;
1117}1392}
1118 1393 
1394+static bool cs35l41_dsm_supported(acpi_handle handle, unsigned int commands)
1395+{
1396+ guid_t guid;
1397+ 
1398+ guid_parse(CS35L41_UUID, &guid);
1399+ 
1400+ return acpi_check_dsm(handle, &guid, 0, BIT(commands));
1401+}
1402+ 
1403+static int cs35l41_get_acpi_mute_state(struct cs35l41_hda *cs35l41, acpi_handle handle)
1404+{
1405+ guid_t guid;
1406+ union acpi_object *ret;
1407+ int mute = -ENODEV;
1408+ 
1409+ guid_parse(CS35L41_UUID, &guid);
1410+ 
1411+ if (cs35l41_dsm_supported(handle, CS35L41_DSM_GET_MUTE)) {
1412+ ret = acpi_evaluate_dsm(handle, &guid, 0, CS35L41_DSM_GET_MUTE, NULL);
1413+ if (!ret)
1414+ return -EINVAL;
1415+ mute = *ret->buffer.pointer;
1416+ dev_dbg(cs35l41->dev, "CS35L41_DSM_GET_MUTE: %d\n", mute);
1417+ }
1418+ 
1419+ dev_dbg(cs35l41->dev, "%s: %d\n", __func__, mute);
1420+ 
1421+ return mute;
1422+}
1423+ 
1424+static void cs35l41_acpi_device_notify(acpi_handle handle, u32 event, struct device *dev)
1425+{
1426+ struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
1427+ int mute;
1428+ 
1429+ if (event != CS35L41_NOTIFY_EVENT)
1430+ return;
1431+ 
1432+ mute = cs35l41_get_acpi_mute_state(cs35l41, handle);
1433+ if (mute < 0) {
1434+ dev_warn(cs35l41->dev, "Unable to retrieve mute state: %d\n", mute);
1435+ return;
1436+ }
1437+ 
1438+ dev_dbg(cs35l41->dev, "Requesting mute value: %d\n", mute);
1439+ cs35l41->mute_override = (mute > 0);
1440+ cs35l41_mute(cs35l41->dev, cs35l41->mute_override);
1441+}
1442+ 
1119static int cs35l41_hda_bind(struct device *dev, struct device *master, void *master_data)1443static int cs35l41_hda_bind(struct device *dev, struct device *master, void *master_data)
1120{1444{
1121 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);1445 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
1122- struct hda_component *comps = master_data;1446+ struct hda_component_parent *parent = master_data;
1447+ struct hda_component *comp;
1123 unsigned int sleep_flags;1448 unsigned int sleep_flags;
1124 int ret = 0;1449 int ret = 0;
1125 1450 
1126- if (!comps || cs35l41->index < 0 || cs35l41->index >= HDA_MAX_COMPONENTS)1451+ comp = hda_component_from_index(parent, cs35l41->index);
1452+ if (!comp)
1127 return -EINVAL;1453 return -EINVAL;
1128 1454 
1129- comps = &comps[cs35l41->index];1455+ if (comp->dev)
1130- if (comps->dev)
1131 return -EBUSY;1456 return -EBUSY;
1132 1457 
1133 pm_runtime_get_sync(dev);1458 pm_runtime_get_sync(dev);
1134 1459 
1135 mutex_lock(&cs35l41->fw_mutex);1460 mutex_lock(&cs35l41->fw_mutex);
1136 1461 
1137- comps->dev = dev;1462+ comp->dev = dev;
1463+ cs35l41->codec = parent->codec;
1138 if (!cs35l41->acpi_subsystem_id)1464 if (!cs35l41->acpi_subsystem_id)
1139 cs35l41->acpi_subsystem_id = kasprintf(GFP_KERNEL, "%.8x",1465 cs35l41->acpi_subsystem_id = kasprintf(GFP_KERNEL, "%.8x",
1140- comps->codec->core.subsystem_id);1466+ cs35l41->codec->core.subsystem_id);
1141- cs35l41->codec = comps->codec;
1142- strscpy(comps->name, dev_name(dev), sizeof(comps->name));
1143 1467 
1144- cs35l41->firmware_type = HDA_CS_DSP_FW_SPK_PROT;1468+ strscpy(comp->name, dev_name(dev), sizeof(comp->name));
1469+ 
1470+ cs35l41->firmware_type = CS35L41_HDA_FW_SPK_PROT;
1145 1471 
1146 if (firmware_autostart) {1472 if (firmware_autostart) {
1147 dev_dbg(cs35l41->dev, "Firmware Autostart.\n");1473 dev_dbg(cs35l41->dev, "Firmware Autostart.\n");
@@ -1154,26 +1480,33 @@ static int cs35l41_hda_bind(struct device *dev, struct device *master, void *mas
1154 1480 
1155 ret = cs35l41_create_controls(cs35l41);1481 ret = cs35l41_create_controls(cs35l41);
1156 1482 
1157- comps->playback_hook = cs35l41_hda_playback_hook;1483+ comp->playback_hook = cs35l41_hda_playback_hook;
1158- comps->pre_playback_hook = cs35l41_hda_pre_playback_hook;1484+ comp->pre_playback_hook = cs35l41_hda_pre_playback_hook;
1159- comps->post_playback_hook = cs35l41_hda_post_playback_hook;1485+ comp->post_playback_hook = cs35l41_hda_post_playback_hook;
1486+ comp->acpi_notify = cs35l41_acpi_device_notify;
1487+ comp->adev = cs35l41->dacpi;
1488+ 
1489+ comp->acpi_notifications_supported = cs35l41_dsm_supported(acpi_device_handle(comp->adev),
1490+ CS35L41_DSM_GET_MUTE);
1491+ 
1492+ cs35l41->mute_override = cs35l41_get_acpi_mute_state(cs35l41,
1493+ acpi_device_handle(cs35l41->dacpi)) > 0;
1160 1494 
1161 mutex_unlock(&cs35l41->fw_mutex);1495 mutex_unlock(&cs35l41->fw_mutex);
1162 1496 
1163 sleep_flags = lock_system_sleep();1497 sleep_flags = lock_system_sleep();
1164- if (!device_link_add(&comps->codec->core.dev, cs35l41->dev, DL_FLAG_STATELESS))1498+ if (!device_link_add(&cs35l41->codec->core.dev, cs35l41->dev, DL_FLAG_STATELESS))
1165 dev_warn(dev, "Unable to create device link\n");1499 dev_warn(dev, "Unable to create device link\n");
1166 unlock_system_sleep(sleep_flags);1500 unlock_system_sleep(sleep_flags);
1167 1501 
1168- pm_runtime_mark_last_busy(dev);
1169 pm_runtime_put_autosuspend(dev);1502 pm_runtime_put_autosuspend(dev);
1170 1503 
1171 dev_info(cs35l41->dev,1504 dev_info(cs35l41->dev,
1172 "CS35L41 Bound - SSID: %s, BST: %d, VSPK: %d, CH: %c, FW EN: %d, SPKID: %d\n",1505 "CS35L41 Bound - SSID: %s, BST: %d, VSPK: %d, CH: %c, FW EN: %d, SPKID: %d\n",
1173 cs35l41->acpi_subsystem_id, cs35l41->hw_cfg.bst_type,1506 cs35l41->acpi_subsystem_id, cs35l41->hw_cfg.bst_type,
1174 cs35l41->hw_cfg.gpio1.func == CS35l41_VSPK_SWITCH,1507 cs35l41->hw_cfg.gpio1.func == CS35l41_VSPK_SWITCH,
1175- cs35l41->hw_cfg.spk_pos ? 'R' : 'L',1508+ channel_name[cs35l41->hw_cfg.spk_pos],
1176- cs35l41->firmware_running, cs35l41->speaker_id);1509+ cs35l41->cs_dsp.running, cs35l41->speaker_id);
1177 1510 
1178 return ret;1511 return ret;
1179}1512}
@@ -1181,14 +1514,19 @@ static int cs35l41_hda_bind(struct device *dev, struct device *master, void *mas
1181static void cs35l41_hda_unbind(struct device *dev, struct device *master, void *master_data)1514static void cs35l41_hda_unbind(struct device *dev, struct device *master, void *master_data)
1182{1515{
1183 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);1516 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
1184- struct hda_component *comps = master_data;1517+ struct hda_component_parent *parent = master_data;
1518+ struct hda_component *comp;
1185 unsigned int sleep_flags;1519 unsigned int sleep_flags;
1186 1520 
1187- if (comps[cs35l41->index].dev == dev) {1521+ comp = hda_component_from_index(parent, cs35l41->index);
1188- memset(&comps[cs35l41->index], 0, sizeof(*comps));1522+ if (!comp)
1523+ return;
1524+ 
1525+ if (comp->dev == dev) {
1189 sleep_flags = lock_system_sleep();1526 sleep_flags = lock_system_sleep();
1190 device_link_remove(&cs35l41->codec->core.dev, cs35l41->dev);1527 device_link_remove(&cs35l41->codec->core.dev, cs35l41->dev);
1191 unlock_system_sleep(sleep_flags);1528 unlock_system_sleep(sleep_flags);
1529+ memset(comp, 0, sizeof(*comp));
1192 }1530 }
1193}1531}
1194 1532 
@@ -1275,7 +1613,7 @@ static const struct regmap_irq cs35l41_reg_irqs[] = {
1275 CS35L41_REG_IRQ(IRQ1_STATUS1, AMP_SHORT_ERR),1613 CS35L41_REG_IRQ(IRQ1_STATUS1, AMP_SHORT_ERR),
1276};1614};
1277 1615 
1278-static struct regmap_irq_chip cs35l41_regmap_irq_chip = {1616+static const struct regmap_irq_chip cs35l41_regmap_irq_chip = {
1279 .name = "cs35l41 IRQ1 Controller",1617 .name = "cs35l41 IRQ1 Controller",
1280 .status_base = CS35L41_IRQ1_STATUS1,1618 .status_base = CS35L41_IRQ1_STATUS1,
1281 .mask_base = CS35L41_IRQ1_MASK1,1619 .mask_base = CS35L41_IRQ1_MASK1,
@@ -1286,13 +1624,56 @@ static struct regmap_irq_chip cs35l41_regmap_irq_chip = {
1286 .runtime_pm = true,1624 .runtime_pm = true,
1287};1625};
1288 1626 
1627+static void cs35l41_configure_interrupt(struct cs35l41_hda *cs35l41, int irq_pol)
1628+{
1629+ int irq;
1630+ int ret;
1631+ int i;
1632+ 
1633+ if (!cs35l41->irq) {
1634+ dev_warn(cs35l41->dev, "No Interrupt Found");
1635+ goto err;
1636+ }
1637+ 
1638+ ret = devm_regmap_add_irq_chip(cs35l41->dev, cs35l41->regmap, cs35l41->irq,
1639+ IRQF_ONESHOT | IRQF_SHARED | irq_pol,
1640+ 0, &cs35l41_regmap_irq_chip, &cs35l41->irq_data);
1641+ if (ret) {
1642+ dev_dbg(cs35l41->dev, "Unable to add IRQ Chip: %d.", ret);
1643+ goto err;
1644+ }
1645+ 
1646+ for (i = 0; i < ARRAY_SIZE(cs35l41_irqs); i++) {
1647+ irq = regmap_irq_get_virq(cs35l41->irq_data, cs35l41_irqs[i].irq);
1648+ if (irq < 0) {
1649+ ret = irq;
1650+ dev_dbg(cs35l41->dev, "Unable to map IRQ %s: %d.", cs35l41_irqs[i].name,
1651+ ret);
1652+ goto err;
1653+ }
1654+ 
1655+ ret = devm_request_threaded_irq(cs35l41->dev, irq, NULL,
1656+ cs35l41_irqs[i].handler,
1657+ IRQF_ONESHOT | IRQF_SHARED | irq_pol,
1658+ cs35l41_irqs[i].name, cs35l41);
1659+ if (ret) {
1660+ dev_dbg(cs35l41->dev, "Unable to allocate IRQ %s:: %d.",
1661+ cs35l41_irqs[i].name, ret);
1662+ goto err;
1663+ }
1664+ }
1665+ return;
1666+err:
1667+ dev_warn(cs35l41->dev,
1668+ "IRQ Config Failed. Amp errors may not be recoverable without reboot.");
1669+}
1670+ 
1289static int cs35l41_hda_apply_properties(struct cs35l41_hda *cs35l41)1671static int cs35l41_hda_apply_properties(struct cs35l41_hda *cs35l41)
1290{1672{
1291 struct cs35l41_hw_cfg *hw_cfg = &cs35l41->hw_cfg;1673 struct cs35l41_hw_cfg *hw_cfg = &cs35l41->hw_cfg;
1292 bool using_irq = false;1674 bool using_irq = false;
1293- int irq, irq_pol;1675+ int irq_pol;
1294 int ret;1676 int ret;
1295- int i;
1296 1677 
1297 if (!cs35l41->hw_cfg.valid)1678 if (!cs35l41->hw_cfg.valid)
1298 return -EINVAL;1679 return -EINVAL;
@@ -1335,28 +1716,10 @@ static int cs35l41_hda_apply_properties(struct cs35l41_hda *cs35l41)
1335 1716 
1336 irq_pol = cs35l41_gpio_config(cs35l41->regmap, hw_cfg);1717 irq_pol = cs35l41_gpio_config(cs35l41->regmap, hw_cfg);
1337 1718 
1338- if (cs35l41->irq && using_irq) {1719+ if (using_irq)
1339- ret = devm_regmap_add_irq_chip(cs35l41->dev, cs35l41->regmap, cs35l41->irq,1720+ cs35l41_configure_interrupt(cs35l41, irq_pol);
1340- IRQF_ONESHOT | IRQF_SHARED | irq_pol,
1341- 0, &cs35l41_regmap_irq_chip, &cs35l41->irq_data);
1342- if (ret)
1343- return ret;
1344 1721 
1345- for (i = 0; i < ARRAY_SIZE(cs35l41_irqs); i++) {1722+ return cs35l41_hda_channel_map(cs35l41);
1346- irq = regmap_irq_get_virq(cs35l41->irq_data, cs35l41_irqs[i].irq);
1347- if (irq < 0)
1348- return irq;
1349- 
1350- ret = devm_request_threaded_irq(cs35l41->dev, irq, NULL,
1351- cs35l41_irqs[i].handler,
1352- IRQF_ONESHOT | IRQF_SHARED | irq_pol,
1353- cs35l41_irqs[i].name, cs35l41);
1354- if (ret)
1355- return ret;
1356- }
1357- }
1358- 
1359- return cs35l41_hda_channel_map(cs35l41->dev, 0, NULL, 1, &hw_cfg->spk_pos);
1360}1723}
1361 1724 
1362int cs35l41_get_speaker_id(struct device *dev, int amp_index, int num_amps, int fixed_gpio_id)1725int cs35l41_get_speaker_id(struct device *dev, int amp_index, int num_amps, int fixed_gpio_id)
@@ -1413,37 +1776,14 @@ int cs35l41_get_speaker_id(struct device *dev, int amp_index, int num_amps, int
1413 return speaker_id;1776 return speaker_id;
1414}1777}
1415 1778 
1416-static int cs35l41_hda_read_acpi(struct cs35l41_hda *cs35l41, const char *hid, int id)1779+int cs35l41_hda_parse_acpi(struct cs35l41_hda *cs35l41, struct device *physdev, int id)
1417{1780{
1418 struct cs35l41_hw_cfg *hw_cfg = &cs35l41->hw_cfg;1781 struct cs35l41_hw_cfg *hw_cfg = &cs35l41->hw_cfg;
1419 u32 values[HDA_MAX_COMPONENTS];1782 u32 values[HDA_MAX_COMPONENTS];
1420- struct acpi_device *adev;
1421- struct device *physdev;
1422- const char *sub;
1423 char *property;1783 char *property;
1424 size_t nval;1784 size_t nval;
1425 int i, ret;1785 int i, ret;
1426 1786 
1427- adev = acpi_dev_get_first_match_dev(hid, NULL, -1);
1428- if (!adev) {
1429- dev_err(cs35l41->dev, "Failed to find an ACPI device for %s\n", hid);
1430- return -ENODEV;
1431- }
1432- 
1433- physdev = get_device(acpi_get_first_physical_node(adev));
1434- acpi_dev_put(adev);
1435- 
1436- sub = acpi_get_subsystem_id(ACPI_HANDLE(physdev));
1437- if (IS_ERR(sub))
1438- sub = NULL;
1439- cs35l41->acpi_subsystem_id = sub;
1440- 
1441- ret = cs35l41_add_dsd_properties(cs35l41, physdev, id, hid);
1442- if (!ret) {
1443- dev_info(cs35l41->dev, "Using extra _DSD properties, bypassing _DSD in ACPI\n");
1444- goto put_physdev;
1445- }
1446- 
1447 property = "cirrus,dev-index";1787 property = "cirrus,dev-index";
1448 ret = device_property_count_u32(physdev, property);1788 ret = device_property_count_u32(physdev, property);
1449 if (ret <= 0)1789 if (ret <= 0)
@@ -1475,8 +1815,9 @@ static int cs35l41_hda_read_acpi(struct cs35l41_hda *cs35l41, const char *hid, i
1475 /* To use the same release code for all laptop variants we can't use devm_ version of1815 /* To use the same release code for all laptop variants we can't use devm_ version of
1476 * gpiod_get here, as CLSA010* don't have a fully functional bios with an _DSD node1816 * gpiod_get here, as CLSA010* don't have a fully functional bios with an _DSD node
1477 */1817 */
1478- cs35l41->reset_gpio = fwnode_gpiod_get_index(acpi_fwnode_handle(adev), "reset", cs35l41->index,1818+ cs35l41->reset_gpio = fwnode_gpiod_get_index(acpi_fwnode_handle(cs35l41->dacpi), "reset",
1479- GPIOD_OUT_LOW, "cs35l41-reset");1819+ cs35l41->index, GPIOD_OUT_LOW,
1820+ "cs35l41-reset");
1480 1821 
1481 property = "cirrus,speaker-position";1822 property = "cirrus,speaker-position";
1482 ret = device_property_read_u32_array(physdev, property, values, nval);1823 ret = device_property_read_u32_array(physdev, property, values, nval);
@@ -1532,25 +1873,74 @@ static int cs35l41_hda_read_acpi(struct cs35l41_hda *cs35l41, const char *hid, i
1532 hw_cfg->bst_type = CS35L41_EXT_BOOST;1873 hw_cfg->bst_type = CS35L41_EXT_BOOST;
1533 1874 
1534 hw_cfg->valid = true;1875 hw_cfg->valid = true;
1535- put_device(physdev);
1536 1876 
1537 return 0;1877 return 0;
1538- 
1539err:1878err:
1540 dev_err(cs35l41->dev, "Failed property %s: %d\n", property, ret);1879 dev_err(cs35l41->dev, "Failed property %s: %d\n", property, ret);
1541 hw_cfg->valid = false;1880 hw_cfg->valid = false;
1542 hw_cfg->gpio1.valid = false;1881 hw_cfg->gpio1.valid = false;
1543 hw_cfg->gpio2.valid = false;1882 hw_cfg->gpio2.valid = false;
1544-put_physdev:1883+ acpi_dev_put(cs35l41->dacpi);
1545- put_device(physdev);
1546 1884 
1547 return ret;1885 return ret;
1548}1886}
1549 1887 
1550-int cs35l41_hda_probe(struct device *dev, const char *device_name, int id, int irq,1888+static int cs35l41_hda_read_acpi(struct cs35l41_hda *cs35l41, const char *hid, int id)
1551- struct regmap *regmap)
1552{1889{
1553- unsigned int int_sts, regid, reg_revid, mtl_revid, chipid, int_status;1890+ struct acpi_device *adev;
1891+ struct device *physdev;
1892+ struct spi_device *spi;
1893+ const char *sub;
1894+ int ret;
1895+ 
1896+ adev = acpi_dev_get_first_match_dev(hid, NULL, -1);
1897+ if (!adev) {
1898+ dev_err(cs35l41->dev, "Failed to find an ACPI device for %s\n", hid);
1899+ return -ENODEV;
1900+ }
1901+ 
1902+ cs35l41->dacpi = adev;
1903+ physdev = get_device(acpi_get_first_physical_node(adev));
1904+ if (!physdev)
1905+ return -ENODEV;
1906+ 
1907+ sub = acpi_get_subsystem_id(ACPI_HANDLE(physdev));
1908+ if (IS_ERR(sub))
1909+ sub = NULL;
1910+ cs35l41->acpi_subsystem_id = sub;
1911+ 
1912+ ret = cs35l41_add_dsd_properties(cs35l41, physdev, id, hid);
1913+ if (!ret) {
1914+ dev_info(cs35l41->dev, "Using extra _DSD properties, bypassing _DSD in ACPI\n");
1915+ goto out;
1916+ }
1917+ 
1918+ ret = cs35l41_hda_parse_acpi(cs35l41, physdev, id);
1919+ if (ret) {
1920+ put_device(physdev);
1921+ return ret;
1922+ }
1923+out:
1924+ put_device(physdev);
1925+ 
1926+ cs35l41->bypass_fw = false;
1927+ if (cs35l41->control_bus == SPI) {
1928+ spi = to_spi_device(cs35l41->dev);
1929+ if (spi->max_speed_hz < CS35L41_MAX_ACCEPTABLE_SPI_SPEED_HZ) {
1930+ dev_warn(cs35l41->dev,
1931+ "SPI speed is too slow to support firmware download: %d Hz.\n",
1932+ spi->max_speed_hz);
1933+ cs35l41->bypass_fw = true;
1934+ }
1935+ }
1936+ 
1937+ return 0;
1938+}
1939+ 
1940+int cs35l41_hda_probe(struct device *dev, const char *device_name, int id, int irq,
1941+ struct regmap *regmap, enum control_bus control_bus)
1942+{
1943+ unsigned int regid, reg_revid;
1554 struct cs35l41_hda *cs35l41;1944 struct cs35l41_hda *cs35l41;
1555 int ret;1945 int ret;
1556 1946 
@@ -1567,6 +1957,7 @@ int cs35l41_hda_probe(struct device *dev, const char *device_name, int id, int i
1567 cs35l41->dev = dev;1957 cs35l41->dev = dev;
1568 cs35l41->irq = irq;1958 cs35l41->irq = irq;
1569 cs35l41->regmap = regmap;1959 cs35l41->regmap = regmap;
1960+ cs35l41->control_bus = control_bus;
1570 dev_set_drvdata(dev, cs35l41);1961 dev_set_drvdata(dev, cs35l41);
1571 1962 
1572 ret = cs35l41_hda_read_acpi(cs35l41, device_name, id);1963 ret = cs35l41_hda_read_acpi(cs35l41, device_name, id);
@@ -1584,47 +1975,22 @@ int cs35l41_hda_probe(struct device *dev, const char *device_name, int id, int i
1584 }1975 }
1585 }1976 }
1586 if (cs35l41->reset_gpio) {1977 if (cs35l41->reset_gpio) {
1978+ gpiod_set_value_cansleep(cs35l41->reset_gpio, 0);
1587 usleep_range(2000, 2100);1979 usleep_range(2000, 2100);
1588 gpiod_set_value_cansleep(cs35l41->reset_gpio, 1);1980 gpiod_set_value_cansleep(cs35l41->reset_gpio, 1);
1589 }1981 }
1590 1982 
1983+ usleep_range(2000, 2100);
1984+ regmap_write(cs35l41->regmap, CS35L41_SFT_RESET, CS35L41_SOFTWARE_RESET);
1591 usleep_range(2000, 2100);1985 usleep_range(2000, 2100);
1592 1986 
1593- ret = regmap_read_poll_timeout(cs35l41->regmap, CS35L41_IRQ1_STATUS4, int_status,1987+ ret = cs35l41_wait_boot_done(cs35l41);
1594- int_status & CS35L41_OTP_BOOT_DONE, 1000, 100000);1988+ if (ret)
1595- if (ret) {
1596- dev_err(cs35l41->dev, "Failed waiting for OTP_BOOT_DONE: %d\n", ret);
1597 goto err;1989 goto err;
1598- }
1599 1990 
1600- ret = regmap_read(cs35l41->regmap, CS35L41_IRQ1_STATUS3, &int_sts);1991+ ret = cs35l41_verify_id(cs35l41, &regid, &reg_revid);
1601- if (ret || (int_sts & CS35L41_OTP_BOOT_ERR)) {1992+ if (ret)
1602- dev_err(cs35l41->dev, "OTP Boot status %x error: %d\n",
1603- int_sts & CS35L41_OTP_BOOT_ERR, ret);
1604- ret = -EIO;
1605 goto err;1993 goto err;
1606- }
1607- 
1608- ret = regmap_read(cs35l41->regmap, CS35L41_DEVID, &regid);
1609- if (ret) {
1610- dev_err(cs35l41->dev, "Get Device ID failed: %d\n", ret);
1611- goto err;
1612- }
1613- 
1614- ret = regmap_read(cs35l41->regmap, CS35L41_REVID, &reg_revid);
1615- if (ret) {
1616- dev_err(cs35l41->dev, "Get Revision ID failed: %d\n", ret);
1617- goto err;
1618- }
1619- 
1620- mtl_revid = reg_revid & CS35L41_MTLREVID_MASK;
1621- 
1622- chipid = (mtl_revid % 2) ? CS35L41R_CHIP_ID : CS35L41_CHIP_ID;
1623- if (regid != chipid) {
1624- dev_err(cs35l41->dev, "CS35L41 Device ID (%X). Expected ID %X\n", regid, chipid);
1625- ret = -ENODEV;
1626- goto err;
1627- }
1628 1994 
1629 ret = cs35l41_test_key_unlock(cs35l41->dev, cs35l41->regmap);1995 ret = cs35l41_test_key_unlock(cs35l41->dev, cs35l41->regmap);
1630 if (ret)1996 if (ret)
@@ -1636,7 +2002,7 @@ int cs35l41_hda_probe(struct device *dev, const char *device_name, int id, int i
1636 2002 
1637 ret = cs35l41_otp_unpack(cs35l41->dev, cs35l41->regmap);2003 ret = cs35l41_otp_unpack(cs35l41->dev, cs35l41->regmap);
1638 if (ret) {2004 if (ret) {
1639- dev_err(cs35l41->dev, "OTP Unpack failed: %d\n", ret);2005+ dev_err_probe(cs35l41->dev, ret, "OTP Unpack failed\n");
1640 goto err;2006 goto err;
1641 }2007 }
1642 2008 
@@ -1644,17 +2010,17 @@ int cs35l41_hda_probe(struct device *dev, const char *device_name, int id, int i
1644 if (ret)2010 if (ret)
1645 goto err;2011 goto err;
1646 2012 
1647- ret = regmap_multi_reg_write(cs35l41->regmap, cs35l41_hda_mute,2013+ ret = cs35l41_get_calibration(cs35l41);
1648- ARRAY_SIZE(cs35l41_hda_mute));2014+ if (ret && ret != -ENOENT)
1649- if (ret)
1650 goto err;2015 goto err;
1651 2016 
2017+ cs35l41_mute(cs35l41->dev, true);
2018+ 
1652 INIT_WORK(&cs35l41->fw_load_work, cs35l41_fw_load_work);2019 INIT_WORK(&cs35l41->fw_load_work, cs35l41_fw_load_work);
1653 mutex_init(&cs35l41->fw_mutex);2020 mutex_init(&cs35l41->fw_mutex);
1654 2021 
1655 pm_runtime_set_autosuspend_delay(cs35l41->dev, 3000);2022 pm_runtime_set_autosuspend_delay(cs35l41->dev, 3000);
1656 pm_runtime_use_autosuspend(cs35l41->dev);2023 pm_runtime_use_autosuspend(cs35l41->dev);
1657- pm_runtime_mark_last_busy(cs35l41->dev);
1658 pm_runtime_set_active(cs35l41->dev);2024 pm_runtime_set_active(cs35l41->dev);
1659 pm_runtime_get_noresume(cs35l41->dev);2025 pm_runtime_get_noresume(cs35l41->dev);
1660 pm_runtime_enable(cs35l41->dev);2026 pm_runtime_enable(cs35l41->dev);
@@ -1667,7 +2033,7 @@ int cs35l41_hda_probe(struct device *dev, const char *device_name, int id, int i
1667 2033 
1668 ret = component_add(cs35l41->dev, &cs35l41_hda_comp_ops);2034 ret = component_add(cs35l41->dev, &cs35l41_hda_comp_ops);
1669 if (ret) {2035 if (ret) {
1670- dev_err(cs35l41->dev, "Register component failed: %d\n", ret);2036+ dev_err_probe(cs35l41->dev, ret, "Register component failed\n");
1671 goto err_pm;2037 goto err_pm;
1672 }2038 }
1673 2039 
@@ -1684,16 +2050,20 @@ int cs35l41_hda_probe(struct device *dev, const char *device_name, int id, int i
1684 if (cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type))2050 if (cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type))
1685 gpiod_set_value_cansleep(cs35l41->reset_gpio, 0);2051 gpiod_set_value_cansleep(cs35l41->reset_gpio, 0);
1686 gpiod_put(cs35l41->reset_gpio);2052 gpiod_put(cs35l41->reset_gpio);
2053+ gpiod_put(cs35l41->cs_gpio);
2054+ acpi_dev_put(cs35l41->dacpi);
1687 kfree(cs35l41->acpi_subsystem_id);2055 kfree(cs35l41->acpi_subsystem_id);
1688 2056 
1689 return ret;2057 return ret;
1690}2058}
1691-EXPORT_SYMBOL_NS_GPL(cs35l41_hda_probe, SND_HDA_SCODEC_CS35L41);2059+EXPORT_SYMBOL_NS_GPL(cs35l41_hda_probe, "SND_HDA_SCODEC_CS35L41");
1692 2060 
1693void cs35l41_hda_remove(struct device *dev)2061void cs35l41_hda_remove(struct device *dev)
1694{2062{
1695 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);2063 struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
1696 2064 
2065+ component_del(cs35l41->dev, &cs35l41_hda_comp_ops);
2066+ 
1697 pm_runtime_get_sync(cs35l41->dev);2067 pm_runtime_get_sync(cs35l41->dev);
1698 pm_runtime_dont_use_autosuspend(cs35l41->dev);2068 pm_runtime_dont_use_autosuspend(cs35l41->dev);
1699 pm_runtime_disable(cs35l41->dev);2069 pm_runtime_disable(cs35l41->dev);
@@ -1701,16 +2071,17 @@ void cs35l41_hda_remove(struct device *dev)
1701 if (cs35l41->halo_initialized)2071 if (cs35l41->halo_initialized)
1702 cs35l41_remove_dsp(cs35l41);2072 cs35l41_remove_dsp(cs35l41);
1703 2073 
1704- component_del(cs35l41->dev, &cs35l41_hda_comp_ops);2074+ acpi_dev_put(cs35l41->dacpi);
1705 2075 
1706 pm_runtime_put_noidle(cs35l41->dev);2076 pm_runtime_put_noidle(cs35l41->dev);
1707 2077 
1708 if (cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type))2078 if (cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type))
1709 gpiod_set_value_cansleep(cs35l41->reset_gpio, 0);2079 gpiod_set_value_cansleep(cs35l41->reset_gpio, 0);
1710 gpiod_put(cs35l41->reset_gpio);2080 gpiod_put(cs35l41->reset_gpio);
2081+ gpiod_put(cs35l41->cs_gpio);
1711 kfree(cs35l41->acpi_subsystem_id);2082 kfree(cs35l41->acpi_subsystem_id);
1712}2083}
1713-EXPORT_SYMBOL_NS_GPL(cs35l41_hda_remove, SND_HDA_SCODEC_CS35L41);2084+EXPORT_SYMBOL_NS_GPL(cs35l41_hda_remove, "SND_HDA_SCODEC_CS35L41");
1714 2085 
1715const struct dev_pm_ops cs35l41_hda_pm_ops = {2086const struct dev_pm_ops cs35l41_hda_pm_ops = {
1716 RUNTIME_PM_OPS(cs35l41_runtime_suspend, cs35l41_runtime_resume,2087 RUNTIME_PM_OPS(cs35l41_runtime_suspend, cs35l41_runtime_resume,
@@ -1718,10 +2089,12 @@ const struct dev_pm_ops cs35l41_hda_pm_ops = {
1718 .prepare = cs35l41_system_suspend_prep,2089 .prepare = cs35l41_system_suspend_prep,
1719 SYSTEM_SLEEP_PM_OPS(cs35l41_system_suspend, cs35l41_system_resume)2090 SYSTEM_SLEEP_PM_OPS(cs35l41_system_suspend, cs35l41_system_resume)
1720};2091};
1721-EXPORT_SYMBOL_NS_GPL(cs35l41_hda_pm_ops, SND_HDA_SCODEC_CS35L41);2092+EXPORT_SYMBOL_NS_GPL(cs35l41_hda_pm_ops, "SND_HDA_SCODEC_CS35L41");
1722 2093 
1723MODULE_DESCRIPTION("CS35L41 HDA Driver");2094MODULE_DESCRIPTION("CS35L41 HDA Driver");
1724-MODULE_IMPORT_NS(SND_HDA_CS_DSP_CONTROLS);2095+MODULE_IMPORT_NS("SND_SOC_CS_AMP_LIB");
1725MODULE_AUTHOR("Lucas Tanure, Cirrus Logic Inc, <tanureal@opensource.cirrus.com>");2096MODULE_AUTHOR("Lucas Tanure, Cirrus Logic Inc, <tanureal@opensource.cirrus.com>");
1726MODULE_LICENSE("GPL");2097MODULE_LICENSE("GPL");
1727-MODULE_IMPORT_NS(FW_CS_DSP);2098+MODULE_IMPORT_NS("FW_CS_DSP");
2099+MODULE_FIRMWARE("cirrus/cs35l41-*.wmfw");
2100+MODULE_FIRMWARE("cirrus/cs35l41-*.bin");
Msound/soc/intel/avs/pcm.c+8-4
@@ -1171,14 +1171,18 @@ int avs_soc_component_register(struct device *dev, const char *name,
1171 if (!acomp)1171 if (!acomp)
1172 return -ENOMEM;1172 return -ENOMEM;
1173 1173 
1174+ acomp->base.name = devm_kstrdup(dev, name, GFP_KERNEL);
1175+ if (!acomp->base.name)
1176+ return -ENOMEM;
1177+ 
1178+ INIT_LIST_HEAD(&acomp->node);
1179+ 
1180+ drv->use_dai_pcm_id = !obsolete_card_names;
1181+ 
1174 ret = snd_soc_component_initialize(&acomp->base, drv, dev);1182 ret = snd_soc_component_initialize(&acomp->base, drv, dev);
1175 if (ret < 0)1183 if (ret < 0)
1176 return ret;1184 return ret;
1177 1185 
1178- /* force name change after ASoC is done with its init */
1179- acomp->base.name = name;
1180- INIT_LIST_HEAD(&acomp->node);
1181- 
1182 return snd_soc_add_component(&acomp->base, cpu_dais, num_cpu_dais);1186 return snd_soc_add_component(&acomp->base, cpu_dais, num_cpu_dais);
1183}1187}
1184 1188 
Msound/usb/endpoint.c+5-0
@@ -1383,6 +1383,11 @@ int snd_usb_endpoint_set_params(struct snd_usb_audio *chip,
1383 ep->sample_rem = ep->cur_rate % ep->pps;1383 ep->sample_rem = ep->cur_rate % ep->pps;
1384 ep->packsize[0] = ep->cur_rate / ep->pps;1384 ep->packsize[0] = ep->cur_rate / ep->pps;
1385 ep->packsize[1] = (ep->cur_rate + (ep->pps - 1)) / ep->pps;1385 ep->packsize[1] = (ep->cur_rate + (ep->pps - 1)) / ep->pps;
1386+ if (ep->packsize[1] > ep->maxpacksize) {
1387+ usb_audio_dbg(chip, "Too small maxpacksize %u for rate %u / pps %u\n",
1388+ ep->maxpacksize, ep->cur_rate, ep->pps);
1389+ return -EINVAL;
1390+ }
1386 1391 
1387 /* calculate the frequency in 16.16 format */1392 /* calculate the frequency in 16.16 format */
1388 ep->freqm = ep->freqn;1393 ep->freqm = ep->freqn;
Msound/usb/stream.c+22-3
@@ -341,20 +341,28 @@ snd_pcm_chmap_elem *convert_chmap_v3(struct uac3_cluster_header_descriptor
341 341 
342 len = le16_to_cpu(cluster->wLength);342 len = le16_to_cpu(cluster->wLength);
343 c = 0;343 c = 0;
344- p += sizeof(struct uac3_cluster_header_descriptor);344+ p += sizeof(*cluster);
345+ len -= sizeof(*cluster);
345 346 
346- while (((p - (void *)cluster) < len) && (c < channels)) {347+ while (len > 0 && (c < channels)) {
347 struct uac3_cluster_segment_descriptor *cs_desc = p;348 struct uac3_cluster_segment_descriptor *cs_desc = p;
348 u16 cs_len;349 u16 cs_len;
349 u8 cs_type;350 u8 cs_type;
350 351 
352+ if (len < sizeof(*p))
353+ break;
351 cs_len = le16_to_cpu(cs_desc->wLength);354 cs_len = le16_to_cpu(cs_desc->wLength);
355+ if (len < cs_len)
356+ break;
352 cs_type = cs_desc->bSegmentType;357 cs_type = cs_desc->bSegmentType;
353 358 
354 if (cs_type == UAC3_CHANNEL_INFORMATION) {359 if (cs_type == UAC3_CHANNEL_INFORMATION) {
355 struct uac3_cluster_information_segment_descriptor *is = p;360 struct uac3_cluster_information_segment_descriptor *is = p;
356 unsigned char map;361 unsigned char map;
357 362 
363+ if (cs_len < sizeof(*is))
364+ break;
365+ 
358 /*366 /*
359 * TODO: this conversion is not complete, update it367 * TODO: this conversion is not complete, update it
360 * after adding UAC3 values to asound.h368 * after adding UAC3 values to asound.h
@@ -456,6 +464,7 @@ snd_pcm_chmap_elem *convert_chmap_v3(struct uac3_cluster_header_descriptor
456 chmap->map[c++] = map;464 chmap->map[c++] = map;
457 }465 }
458 p += cs_len;466 p += cs_len;
467+ len -= cs_len;
459 }468 }
460 469 
461 if (channels < c)470 if (channels < c)
@@ -876,7 +885,7 @@ snd_usb_get_audioformat_uac3(struct snd_usb_audio *chip,
876 u64 badd_formats = 0;885 u64 badd_formats = 0;
877 unsigned int num_channels;886 unsigned int num_channels;
878 struct audioformat *fp;887 struct audioformat *fp;
879- u16 cluster_id, wLength;888+ u16 cluster_id, wLength, cluster_wLength;
880 int clock = 0;889 int clock = 0;
881 int err;890 int err;
882 891 
@@ -1005,6 +1014,16 @@ snd_usb_get_audioformat_uac3(struct snd_usb_audio *chip,
1005 return ERR_PTR(-EIO);1014 return ERR_PTR(-EIO);
1006 }1015 }
1007 1016 
1017+ cluster_wLength = le16_to_cpu(cluster->wLength);
1018+ if (cluster_wLength < sizeof(*cluster) ||
1019+ cluster_wLength > wLength) {
1020+ dev_err(&dev->dev,
1021+ "%u:%d : invalid Cluster Descriptor size\n",
1022+ iface_no, altno);
1023+ kfree(cluster);
1024+ return ERR_PTR(-EIO);
1025+ }
1026+ 
1008 num_channels = cluster->bNrChannels;1027 num_channels = cluster->bNrChannels;
1009 chmap = convert_chmap_v3(cluster);1028 chmap = convert_chmap_v3(cluster);
1010 kfree(cluster);1029 kfree(cluster);