aboutsummaryrefslogtreecommitdiffstats
path: root/arch/mips/mm
diff options
context:
space:
mode:
authorWe-unite <3205135446@qq.com>2025-03-08 22:04:20 +0800
committerWe-unite <3205135446@qq.com>2025-03-08 22:04:20 +0800
commita07bb8fd1299070229f0e8f3dcb57ffd5ef9870a (patch)
tree84f21bd0bf7071bc5fc7dd989e77d7ceb5476682 /arch/mips/mm
downloadohosKernel-a07bb8fd1299070229f0e8f3dcb57ffd5ef9870a.tar.gz
ohosKernel-a07bb8fd1299070229f0e8f3dcb57ffd5ef9870a.zip
Initial commit: OpenHarmony-v4.0-ReleaseOpenHarmony-v4.0-Release
Diffstat (limited to 'arch/mips/mm')
-rw-r--r--arch/mips/mm/Makefile42
-rw-r--r--arch/mips/mm/c-octeon.c352
-rw-r--r--arch/mips/mm/c-r3k.c319
-rw-r--r--arch/mips/mm/c-r4k.c2009
-rw-r--r--arch/mips/mm/c-tx39.c414
-rw-r--r--arch/mips/mm/cache.c242
-rw-r--r--arch/mips/mm/cerr-sb1.c569
-rw-r--r--arch/mips/mm/cex-gen.S42
-rw-r--r--arch/mips/mm/cex-oct.S70
-rw-r--r--arch/mips/mm/cex-sb1.S157
-rw-r--r--arch/mips/mm/context.c291
-rw-r--r--arch/mips/mm/dma-noncoherent.c145
-rw-r--r--arch/mips/mm/extable.c25
-rw-r--r--arch/mips/mm/fault.c332
-rw-r--r--arch/mips/mm/highmem.c94
-rw-r--r--arch/mips/mm/hugetlbpage.c81
-rw-r--r--arch/mips/mm/init.c581
-rw-r--r--arch/mips/mm/ioremap.c119
-rw-r--r--arch/mips/mm/ioremap64.c23
-rw-r--r--arch/mips/mm/mmap.c129
-rw-r--r--arch/mips/mm/page-funcs.S53
-rw-r--r--arch/mips/mm/page.c681
-rw-r--r--arch/mips/mm/pgtable-32.c91
-rw-r--r--arch/mips/mm/pgtable-64.c125
-rw-r--r--arch/mips/mm/pgtable.c25
-rw-r--r--arch/mips/mm/sc-debugfs.c61
-rw-r--r--arch/mips/mm/sc-ip22.c190
-rw-r--r--arch/mips/mm/sc-mips.c264
-rw-r--r--arch/mips/mm/sc-r5k.c107
-rw-r--r--arch/mips/mm/sc-rm7k.c270
-rw-r--r--arch/mips/mm/tlb-funcs.S40
-rw-r--r--arch/mips/mm/tlb-r3k.c284
-rw-r--r--arch/mips/mm/tlb-r4k.c583
-rw-r--r--arch/mips/mm/tlbex-fault.S28
-rw-r--r--arch/mips/mm/tlbex.c2661
-rw-r--r--arch/mips/mm/uasm-micromips.c232
-rw-r--r--arch/mips/mm/uasm-mips.c290
-rw-r--r--arch/mips/mm/uasm.c643
38 files changed, 12664 insertions, 0 deletions
diff --git a/arch/mips/mm/Makefile b/arch/mips/mm/Makefile
new file mode 100644
index 000000000..865926a37
--- /dev/null
+++ b/arch/mips/mm/Makefile
@@ -0,0 +1,42 @@
1# SPDX-License-Identifier: GPL-2.0
2#
3# Makefile for the Linux/MIPS-specific parts of the memory manager.
4#
5
6obj-y += cache.o
7obj-y += context.o
8obj-y += extable.o
9obj-y += fault.o
10obj-y += init.o
11obj-y += mmap.o
12obj-y += page.o
13obj-y += page-funcs.o
14obj-y += pgtable.o
15obj-y += tlbex.o
16obj-y += tlbex-fault.o
17obj-y += tlb-funcs.o
18
19ifdef CONFIG_CPU_MICROMIPS
20obj-y += uasm-micromips.o
21else
22obj-y += uasm-mips.o
23endif
24
25obj-$(CONFIG_32BIT) += ioremap.o pgtable-32.o
26obj-$(CONFIG_64BIT) += ioremap64.o pgtable-64.o
27obj-$(CONFIG_HIGHMEM) += highmem.o
28obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
29obj-$(CONFIG_DMA_NONCOHERENT) += dma-noncoherent.o
30
31obj-$(CONFIG_CPU_R3K_TLB) += tlb-r3k.o
32obj-$(CONFIG_CPU_R4K_CACHE_TLB) += c-r4k.o cex-gen.o tlb-r4k.o
33obj-$(CONFIG_CPU_R3000) += c-r3k.o
34obj-$(CONFIG_CPU_SB1) += c-r4k.o cerr-sb1.o cex-sb1.o tlb-r4k.o
35obj-$(CONFIG_CPU_TX39XX) += c-tx39.o
36obj-$(CONFIG_CPU_CAVIUM_OCTEON) += c-octeon.o cex-oct.o tlb-r4k.o
37
38obj-$(CONFIG_IP22_CPU_SCACHE) += sc-ip22.o
39obj-$(CONFIG_R5000_CPU_SCACHE) += sc-r5k.o
40obj-$(CONFIG_RM7000_CPU_SCACHE) += sc-rm7k.o
41obj-$(CONFIG_MIPS_CPU_SCACHE) += sc-mips.o
42obj-$(CONFIG_SCACHE_DEBUGFS) += sc-debugfs.o
diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c
new file mode 100644
index 000000000..8ae181e08
--- /dev/null
+++ b/arch/mips/mm/c-octeon.c
@@ -0,0 +1,352 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2005-2007 Cavium Networks
7 */
8#include <linux/export.h>
9#include <linux/kernel.h>
10#include <linux/sched.h>
11#include <linux/smp.h>
12#include <linux/mm.h>
13#include <linux/bitops.h>
14#include <linux/cpu.h>
15#include <linux/io.h>
16
17#include <asm/bcache.h>
18#include <asm/bootinfo.h>
19#include <asm/cacheops.h>
20#include <asm/cpu-features.h>
21#include <asm/cpu-type.h>
22#include <asm/page.h>
23#include <asm/r4kcache.h>
24#include <asm/traps.h>
25#include <asm/mmu_context.h>
26#include <asm/war.h>
27
28#include <asm/octeon/octeon.h>
29
30unsigned long long cache_err_dcache[NR_CPUS];
31EXPORT_SYMBOL_GPL(cache_err_dcache);
32
33/**
34 * Octeon automatically flushes the dcache on tlb changes, so
35 * from Linux's viewpoint it acts much like a physically
36 * tagged cache. No flushing is needed
37 *
38 */
39static void octeon_flush_data_cache_page(unsigned long addr)
40{
41 /* Nothing to do */
42}
43
44static inline void octeon_local_flush_icache(void)
45{
46 asm volatile ("synci 0($0)");
47}
48
49/*
50 * Flush local I-cache for the specified range.
51 */
52static void local_octeon_flush_icache_range(unsigned long start,
53 unsigned long end)
54{
55 octeon_local_flush_icache();
56}
57
58/**
59 * Flush caches as necessary for all cores affected by a
60 * vma. If no vma is supplied, all cores are flushed.
61 *
62 * @vma: VMA to flush or NULL to flush all icaches.
63 */
64static void octeon_flush_icache_all_cores(struct vm_area_struct *vma)
65{
66 extern void octeon_send_ipi_single(int cpu, unsigned int action);
67#ifdef CONFIG_SMP
68 int cpu;
69 cpumask_t mask;
70#endif
71
72 mb();
73 octeon_local_flush_icache();
74#ifdef CONFIG_SMP
75 preempt_disable();
76 cpu = smp_processor_id();
77
78 /*
79 * If we have a vma structure, we only need to worry about
80 * cores it has been used on
81 */
82 if (vma)
83 mask = *mm_cpumask(vma->vm_mm);
84 else
85 mask = *cpu_online_mask;
86 cpumask_clear_cpu(cpu, &mask);
87 for_each_cpu(cpu, &mask)
88 octeon_send_ipi_single(cpu, SMP_ICACHE_FLUSH);
89
90 preempt_enable();
91#endif
92}
93
94
95/**
96 * Called to flush the icache on all cores
97 */
98static void octeon_flush_icache_all(void)
99{
100 octeon_flush_icache_all_cores(NULL);
101}
102
103
104/**
105 * Called to flush all memory associated with a memory
106 * context.
107 *
108 * @mm: Memory context to flush
109 */
110static void octeon_flush_cache_mm(struct mm_struct *mm)
111{
112 /*
113 * According to the R4K version of this file, CPUs without
114 * dcache aliases don't need to do anything here
115 */
116}
117
118
119/**
120 * Flush a range of kernel addresses out of the icache
121 *
122 */
123static void octeon_flush_icache_range(unsigned long start, unsigned long end)
124{
125 octeon_flush_icache_all_cores(NULL);
126}
127
128
129/**
130 * Flush a range out of a vma
131 *
132 * @vma: VMA to flush
133 * @start:
134 * @end:
135 */
136static void octeon_flush_cache_range(struct vm_area_struct *vma,
137 unsigned long start, unsigned long end)
138{
139 if (vma->vm_flags & VM_EXEC)
140 octeon_flush_icache_all_cores(vma);
141}
142
143
144/**
145 * Flush a specific page of a vma
146 *
147 * @vma: VMA to flush page for
148 * @page: Page to flush
149 * @pfn:
150 */
151static void octeon_flush_cache_page(struct vm_area_struct *vma,
152 unsigned long page, unsigned long pfn)
153{
154 if (vma->vm_flags & VM_EXEC)
155 octeon_flush_icache_all_cores(vma);
156}
157
158static void octeon_flush_kernel_vmap_range(unsigned long vaddr, int size)
159{
160 BUG();
161}
162
163/**
164 * Probe Octeon's caches
165 *
166 */
167static void probe_octeon(void)
168{
169 unsigned long icache_size;
170 unsigned long dcache_size;
171 unsigned int config1;
172 struct cpuinfo_mips *c = &current_cpu_data;
173 int cputype = current_cpu_type();
174
175 config1 = read_c0_config1();
176 switch (cputype) {
177 case CPU_CAVIUM_OCTEON:
178 case CPU_CAVIUM_OCTEON_PLUS:
179 c->icache.linesz = 2 << ((config1 >> 19) & 7);
180 c->icache.sets = 64 << ((config1 >> 22) & 7);
181 c->icache.ways = 1 + ((config1 >> 16) & 7);
182 c->icache.flags |= MIPS_CACHE_VTAG;
183 icache_size =
184 c->icache.sets * c->icache.ways * c->icache.linesz;
185 c->icache.waybit = ffs(icache_size / c->icache.ways) - 1;
186 c->dcache.linesz = 128;
187 if (cputype == CPU_CAVIUM_OCTEON_PLUS)
188 c->dcache.sets = 2; /* CN5XXX has two Dcache sets */
189 else
190 c->dcache.sets = 1; /* CN3XXX has one Dcache set */
191 c->dcache.ways = 64;
192 dcache_size =
193 c->dcache.sets * c->dcache.ways * c->dcache.linesz;
194 c->dcache.waybit = ffs(dcache_size / c->dcache.ways) - 1;
195 c->options |= MIPS_CPU_PREFETCH;
196 break;
197
198 case CPU_CAVIUM_OCTEON2:
199 c->icache.linesz = 2 << ((config1 >> 19) & 7);
200 c->icache.sets = 8;
201 c->icache.ways = 37;
202 c->icache.flags |= MIPS_CACHE_VTAG;
203 icache_size = c->icache.sets * c->icache.ways * c->icache.linesz;
204
205 c->dcache.linesz = 128;
206 c->dcache.ways = 32;
207 c->dcache.sets = 8;
208 dcache_size = c->dcache.sets * c->dcache.ways * c->dcache.linesz;
209 c->options |= MIPS_CPU_PREFETCH;
210 break;
211
212 case CPU_CAVIUM_OCTEON3:
213 c->icache.linesz = 128;
214 c->icache.sets = 16;
215 c->icache.ways = 39;
216 c->icache.flags |= MIPS_CACHE_VTAG;
217 icache_size = c->icache.sets * c->icache.ways * c->icache.linesz;
218
219 c->dcache.linesz = 128;
220 c->dcache.ways = 32;
221 c->dcache.sets = 8;
222 dcache_size = c->dcache.sets * c->dcache.ways * c->dcache.linesz;
223 c->options |= MIPS_CPU_PREFETCH;
224 break;
225
226 default:
227 panic("Unsupported Cavium Networks CPU type");
228 break;
229 }
230
231 /* compute a couple of other cache variables */
232 c->icache.waysize = icache_size / c->icache.ways;
233 c->dcache.waysize = dcache_size / c->dcache.ways;
234
235 c->icache.sets = icache_size / (c->icache.linesz * c->icache.ways);
236 c->dcache.sets = dcache_size / (c->dcache.linesz * c->dcache.ways);
237
238 if (smp_processor_id() == 0) {
239 pr_info("Primary instruction cache %ldkB, %s, %d way, "
240 "%d sets, linesize %d bytes.\n",
241 icache_size >> 10,
242 cpu_has_vtag_icache ?
243 "virtually tagged" : "physically tagged",
244 c->icache.ways, c->icache.sets, c->icache.linesz);
245
246 pr_info("Primary data cache %ldkB, %d-way, %d sets, "
247 "linesize %d bytes.\n",
248 dcache_size >> 10, c->dcache.ways,
249 c->dcache.sets, c->dcache.linesz);
250 }
251}
252
253static void octeon_cache_error_setup(void)
254{
255 extern char except_vec2_octeon;
256 set_handler(0x100, &except_vec2_octeon, 0x80);
257}
258
259/**
260 * Setup the Octeon cache flush routines
261 *
262 */
263void octeon_cache_init(void)
264{
265 probe_octeon();
266
267 shm_align_mask = PAGE_SIZE - 1;
268
269 flush_cache_all = octeon_flush_icache_all;
270 __flush_cache_all = octeon_flush_icache_all;
271 flush_cache_mm = octeon_flush_cache_mm;
272 flush_cache_page = octeon_flush_cache_page;
273 flush_cache_range = octeon_flush_cache_range;
274 flush_icache_all = octeon_flush_icache_all;
275 flush_data_cache_page = octeon_flush_data_cache_page;
276 flush_icache_range = octeon_flush_icache_range;
277 local_flush_icache_range = local_octeon_flush_icache_range;
278 __flush_icache_user_range = octeon_flush_icache_range;
279 __local_flush_icache_user_range = local_octeon_flush_icache_range;
280
281 __flush_kernel_vmap_range = octeon_flush_kernel_vmap_range;
282
283 build_clear_page();
284 build_copy_page();
285
286 board_cache_error_setup = octeon_cache_error_setup;
287}
288
289/*
290 * Handle a cache error exception
291 */
292static RAW_NOTIFIER_HEAD(co_cache_error_chain);
293
294int register_co_cache_error_notifier(struct notifier_block *nb)
295{
296 return raw_notifier_chain_register(&co_cache_error_chain, nb);
297}
298EXPORT_SYMBOL_GPL(register_co_cache_error_notifier);
299
300int unregister_co_cache_error_notifier(struct notifier_block *nb)
301{
302 return raw_notifier_chain_unregister(&co_cache_error_chain, nb);
303}
304EXPORT_SYMBOL_GPL(unregister_co_cache_error_notifier);
305
306static void co_cache_error_call_notifiers(unsigned long val)
307{
308 int rv = raw_notifier_call_chain(&co_cache_error_chain, val, NULL);
309 if ((rv & ~NOTIFY_STOP_MASK) != NOTIFY_OK) {
310 u64 dcache_err;
311 unsigned long coreid = cvmx_get_core_num();
312 u64 icache_err = read_octeon_c0_icacheerr();
313
314 if (val) {
315 dcache_err = cache_err_dcache[coreid];
316 cache_err_dcache[coreid] = 0;
317 } else {
318 dcache_err = read_octeon_c0_dcacheerr();
319 }
320
321 pr_err("Core%lu: Cache error exception:\n", coreid);
322 pr_err("cp0_errorepc == %lx\n", read_c0_errorepc());
323 if (icache_err & 1) {
324 pr_err("CacheErr (Icache) == %llx\n",
325 (unsigned long long)icache_err);
326 write_octeon_c0_icacheerr(0);
327 }
328 if (dcache_err & 1) {
329 pr_err("CacheErr (Dcache) == %llx\n",
330 (unsigned long long)dcache_err);
331 }
332 }
333}
334
335/*
336 * Called when the the exception is recoverable
337 */
338
339asmlinkage void cache_parity_error_octeon_recoverable(void)
340{
341 co_cache_error_call_notifiers(0);
342}
343
344/**
345 * Called when the the exception is not recoverable
346 */
347
348asmlinkage void cache_parity_error_octeon_non_recoverable(void)
349{
350 co_cache_error_call_notifiers(1);
351 panic("Can't handle cache error: nested exception");
352}
diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c
new file mode 100644
index 000000000..df6755ca1
--- /dev/null
+++ b/arch/mips/mm/c-r3k.c
@@ -0,0 +1,319 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * r2300.c: R2000 and R3000 specific mmu/cache code.
4 *
5 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
6 *
7 * with a lot of changes to make this thing work for R3000s
8 * Tx39XX R4k style caches added. HK
9 * Copyright (C) 1998, 1999, 2000 Harald Koerfgen
10 * Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
11 * Copyright (C) 2001, 2004, 2007 Maciej W. Rozycki
12 */
13#include <linux/kernel.h>
14#include <linux/sched.h>
15#include <linux/smp.h>
16#include <linux/mm.h>
17
18#include <asm/page.h>
19#include <asm/mmu_context.h>
20#include <asm/isadep.h>
21#include <asm/io.h>
22#include <asm/bootinfo.h>
23#include <asm/cpu.h>
24
25static unsigned long icache_size, dcache_size; /* Size in bytes */
26static unsigned long icache_lsize, dcache_lsize; /* Size in bytes */
27
28unsigned long r3k_cache_size(unsigned long ca_flags)
29{
30 unsigned long flags, status, dummy, size;
31 volatile unsigned long *p;
32
33 p = (volatile unsigned long *) KSEG0;
34
35 flags = read_c0_status();
36
37 /* isolate cache space */
38 write_c0_status((ca_flags|flags)&~ST0_IEC);
39
40 *p = 0xa5a55a5a;
41 dummy = *p;
42 status = read_c0_status();
43
44 if (dummy != 0xa5a55a5a || (status & ST0_CM)) {
45 size = 0;
46 } else {
47 for (size = 128; size <= 0x40000; size <<= 1)
48 *(p + size) = 0;
49 *p = -1;
50 for (size = 128;
51 (size <= 0x40000) && (*(p + size) == 0);
52 size <<= 1)
53 ;
54 if (size > 0x40000)
55 size = 0;
56 }
57
58 write_c0_status(flags);
59
60 return size * sizeof(*p);
61}
62
63unsigned long r3k_cache_lsize(unsigned long ca_flags)
64{
65 unsigned long flags, status, lsize, i;
66 volatile unsigned long *p;
67
68 p = (volatile unsigned long *) KSEG0;
69
70 flags = read_c0_status();
71
72 /* isolate cache space */
73 write_c0_status((ca_flags|flags)&~ST0_IEC);
74
75 for (i = 0; i < 128; i++)
76 *(p + i) = 0;
77 *(volatile unsigned char *)p = 0;
78 for (lsize = 1; lsize < 128; lsize <<= 1) {
79 *(p + lsize);
80 status = read_c0_status();
81 if (!(status & ST0_CM))
82 break;
83 }
84 for (i = 0; i < 128; i += lsize)
85 *(volatile unsigned char *)(p + i) = 0;
86
87 write_c0_status(flags);
88
89 return lsize * sizeof(*p);
90}
91
92static void r3k_probe_cache(void)
93{
94 dcache_size = r3k_cache_size(ST0_ISC);
95 if (dcache_size)
96 dcache_lsize = r3k_cache_lsize(ST0_ISC);
97
98 icache_size = r3k_cache_size(ST0_ISC|ST0_SWC);
99 if (icache_size)
100 icache_lsize = r3k_cache_lsize(ST0_ISC|ST0_SWC);
101}
102
103static void r3k_flush_icache_range(unsigned long start, unsigned long end)
104{
105 unsigned long size, i, flags;
106 volatile unsigned char *p;
107
108 size = end - start;
109 if (size > icache_size || KSEGX(start) != KSEG0) {
110 start = KSEG0;
111 size = icache_size;
112 }
113 p = (char *)start;
114
115 flags = read_c0_status();
116
117 /* isolate cache space */
118 write_c0_status((ST0_ISC|ST0_SWC|flags)&~ST0_IEC);
119
120 for (i = 0; i < size; i += 0x080) {
121 asm( "sb\t$0, 0x000(%0)\n\t"
122 "sb\t$0, 0x004(%0)\n\t"
123 "sb\t$0, 0x008(%0)\n\t"
124 "sb\t$0, 0x00c(%0)\n\t"
125 "sb\t$0, 0x010(%0)\n\t"
126 "sb\t$0, 0x014(%0)\n\t"
127 "sb\t$0, 0x018(%0)\n\t"
128 "sb\t$0, 0x01c(%0)\n\t"
129 "sb\t$0, 0x020(%0)\n\t"
130 "sb\t$0, 0x024(%0)\n\t"
131 "sb\t$0, 0x028(%0)\n\t"
132 "sb\t$0, 0x02c(%0)\n\t"
133 "sb\t$0, 0x030(%0)\n\t"
134 "sb\t$0, 0x034(%0)\n\t"
135 "sb\t$0, 0x038(%0)\n\t"
136 "sb\t$0, 0x03c(%0)\n\t"
137 "sb\t$0, 0x040(%0)\n\t"
138 "sb\t$0, 0x044(%0)\n\t"
139 "sb\t$0, 0x048(%0)\n\t"
140 "sb\t$0, 0x04c(%0)\n\t"
141 "sb\t$0, 0x050(%0)\n\t"
142 "sb\t$0, 0x054(%0)\n\t"
143 "sb\t$0, 0x058(%0)\n\t"
144 "sb\t$0, 0x05c(%0)\n\t"
145 "sb\t$0, 0x060(%0)\n\t"
146 "sb\t$0, 0x064(%0)\n\t"
147 "sb\t$0, 0x068(%0)\n\t"
148 "sb\t$0, 0x06c(%0)\n\t"
149 "sb\t$0, 0x070(%0)\n\t"
150 "sb\t$0, 0x074(%0)\n\t"
151 "sb\t$0, 0x078(%0)\n\t"
152 "sb\t$0, 0x07c(%0)\n\t"
153 : : "r" (p) );
154 p += 0x080;
155 }
156
157 write_c0_status(flags);
158}
159
160static void r3k_flush_dcache_range(unsigned long start, unsigned long end)
161{
162 unsigned long size, i, flags;
163 volatile unsigned char *p;
164
165 size = end - start;
166 if (size > dcache_size || KSEGX(start) != KSEG0) {
167 start = KSEG0;
168 size = dcache_size;
169 }
170 p = (char *)start;
171
172 flags = read_c0_status();
173
174 /* isolate cache space */
175 write_c0_status((ST0_ISC|flags)&~ST0_IEC);
176
177 for (i = 0; i < size; i += 0x080) {
178 asm( "sb\t$0, 0x000(%0)\n\t"
179 "sb\t$0, 0x004(%0)\n\t"
180 "sb\t$0, 0x008(%0)\n\t"
181 "sb\t$0, 0x00c(%0)\n\t"
182 "sb\t$0, 0x010(%0)\n\t"
183 "sb\t$0, 0x014(%0)\n\t"
184 "sb\t$0, 0x018(%0)\n\t"
185 "sb\t$0, 0x01c(%0)\n\t"
186 "sb\t$0, 0x020(%0)\n\t"
187 "sb\t$0, 0x024(%0)\n\t"
188 "sb\t$0, 0x028(%0)\n\t"
189 "sb\t$0, 0x02c(%0)\n\t"
190 "sb\t$0, 0x030(%0)\n\t"
191 "sb\t$0, 0x034(%0)\n\t"
192 "sb\t$0, 0x038(%0)\n\t"
193 "sb\t$0, 0x03c(%0)\n\t"
194 "sb\t$0, 0x040(%0)\n\t"
195 "sb\t$0, 0x044(%0)\n\t"
196 "sb\t$0, 0x048(%0)\n\t"
197 "sb\t$0, 0x04c(%0)\n\t"
198 "sb\t$0, 0x050(%0)\n\t"
199 "sb\t$0, 0x054(%0)\n\t"
200 "sb\t$0, 0x058(%0)\n\t"
201 "sb\t$0, 0x05c(%0)\n\t"
202 "sb\t$0, 0x060(%0)\n\t"
203 "sb\t$0, 0x064(%0)\n\t"
204 "sb\t$0, 0x068(%0)\n\t"
205 "sb\t$0, 0x06c(%0)\n\t"
206 "sb\t$0, 0x070(%0)\n\t"
207 "sb\t$0, 0x074(%0)\n\t"
208 "sb\t$0, 0x078(%0)\n\t"
209 "sb\t$0, 0x07c(%0)\n\t"
210 : : "r" (p) );
211 p += 0x080;
212 }
213
214 write_c0_status(flags);
215}
216
217static inline void r3k_flush_cache_all(void)
218{
219}
220
221static inline void r3k___flush_cache_all(void)
222{
223 r3k_flush_dcache_range(KSEG0, KSEG0 + dcache_size);
224 r3k_flush_icache_range(KSEG0, KSEG0 + icache_size);
225}
226
227static void r3k_flush_cache_mm(struct mm_struct *mm)
228{
229}
230
231static void r3k_flush_cache_range(struct vm_area_struct *vma,
232 unsigned long start, unsigned long end)
233{
234}
235
236static void r3k_flush_cache_page(struct vm_area_struct *vma,
237 unsigned long addr, unsigned long pfn)
238{
239 unsigned long kaddr = KSEG0ADDR(pfn << PAGE_SHIFT);
240 int exec = vma->vm_flags & VM_EXEC;
241 struct mm_struct *mm = vma->vm_mm;
242 pmd_t *pmdp;
243 pte_t *ptep;
244
245 pr_debug("cpage[%08llx,%08lx]\n",
246 cpu_context(smp_processor_id(), mm), addr);
247
248 /* No ASID => no such page in the cache. */
249 if (cpu_context(smp_processor_id(), mm) == 0)
250 return;
251
252 pmdp = pmd_off(mm, addr);
253 ptep = pte_offset_kernel(pmdp, addr);
254
255 /* Invalid => no such page in the cache. */
256 if (!(pte_val(*ptep) & _PAGE_PRESENT))
257 return;
258
259 r3k_flush_dcache_range(kaddr, kaddr + PAGE_SIZE);
260 if (exec)
261 r3k_flush_icache_range(kaddr, kaddr + PAGE_SIZE);
262}
263
264static void local_r3k_flush_data_cache_page(void *addr)
265{
266}
267
268static void r3k_flush_data_cache_page(unsigned long addr)
269{
270}
271
272static void r3k_flush_kernel_vmap_range(unsigned long vaddr, int size)
273{
274 BUG();
275}
276
277static void r3k_dma_cache_wback_inv(unsigned long start, unsigned long size)
278{
279 /* Catch bad driver code */
280 BUG_ON(size == 0);
281
282 iob();
283 r3k_flush_dcache_range(start, start + size);
284}
285
286void r3k_cache_init(void)
287{
288 extern void build_clear_page(void);
289 extern void build_copy_page(void);
290
291 r3k_probe_cache();
292
293 flush_cache_all = r3k_flush_cache_all;
294 __flush_cache_all = r3k___flush_cache_all;
295 flush_cache_mm = r3k_flush_cache_mm;
296 flush_cache_range = r3k_flush_cache_range;
297 flush_cache_page = r3k_flush_cache_page;
298 flush_icache_range = r3k_flush_icache_range;
299 local_flush_icache_range = r3k_flush_icache_range;
300 __flush_icache_user_range = r3k_flush_icache_range;
301 __local_flush_icache_user_range = r3k_flush_icache_range;
302
303 __flush_kernel_vmap_range = r3k_flush_kernel_vmap_range;
304
305 local_flush_data_cache_page = local_r3k_flush_data_cache_page;
306 flush_data_cache_page = r3k_flush_data_cache_page;
307
308 _dma_cache_wback_inv = r3k_dma_cache_wback_inv;
309 _dma_cache_wback = r3k_dma_cache_wback_inv;
310 _dma_cache_inv = r3k_dma_cache_wback_inv;
311
312 pr_info("Primary instruction cache %ldkB, linesize %ld bytes.\n",
313 icache_size >> 10, icache_lsize);
314 pr_info("Primary data cache %ldkB, linesize %ld bytes.\n",
315 dcache_size >> 10, dcache_lsize);
316
317 build_clear_page();
318 build_copy_page();
319}
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
new file mode 100644
index 000000000..96adc3d23
--- /dev/null
+++ b/arch/mips/mm/c-r4k.c
@@ -0,0 +1,2009 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org)
8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
9 */
10#include <linux/cpu_pm.h>
11#include <linux/hardirq.h>
12#include <linux/init.h>
13#include <linux/highmem.h>
14#include <linux/kernel.h>
15#include <linux/linkage.h>
16#include <linux/preempt.h>
17#include <linux/sched.h>
18#include <linux/smp.h>
19#include <linux/mm.h>
20#include <linux/export.h>
21#include <linux/bitops.h>
22
23#include <asm/bcache.h>
24#include <asm/bootinfo.h>
25#include <asm/cache.h>
26#include <asm/cacheops.h>
27#include <asm/cpu.h>
28#include <asm/cpu-features.h>
29#include <asm/cpu-type.h>
30#include <asm/io.h>
31#include <asm/page.h>
32#include <asm/r4kcache.h>
33#include <asm/sections.h>
34#include <asm/mmu_context.h>
35#include <asm/war.h>
36#include <asm/cacheflush.h> /* for run_uncached() */
37#include <asm/traps.h>
38#include <asm/dma-coherence.h>
39#include <asm/mips-cps.h>
40
41/*
42 * Bits describing what cache ops an SMP callback function may perform.
43 *
44 * R4K_HIT - Virtual user or kernel address based cache operations. The
45 * active_mm must be checked before using user addresses, falling
46 * back to kmap.
47 * R4K_INDEX - Index based cache operations.
48 */
49
50#define R4K_HIT BIT(0)
51#define R4K_INDEX BIT(1)
52
53/**
54 * r4k_op_needs_ipi() - Decide if a cache op needs to be done on every core.
55 * @type: Type of cache operations (R4K_HIT or R4K_INDEX).
56 *
57 * Decides whether a cache op needs to be performed on every core in the system.
58 * This may change depending on the @type of cache operation, as well as the set
59 * of online CPUs, so preemption should be disabled by the caller to prevent CPU
60 * hotplug from changing the result.
61 *
62 * Returns: 1 if the cache operation @type should be done on every core in
63 * the system.
64 * 0 if the cache operation @type is globalized and only needs to
65 * be performed on a simple CPU.
66 */
67static inline bool r4k_op_needs_ipi(unsigned int type)
68{
69 /* The MIPS Coherence Manager (CM) globalizes address-based cache ops */
70 if (type == R4K_HIT && mips_cm_present())
71 return false;
72
73 /*
74 * Hardware doesn't globalize the required cache ops, so SMP calls may
75 * be needed, but only if there are foreign CPUs (non-siblings with
76 * separate caches).
77 */
78 /* cpu_foreign_map[] undeclared when !CONFIG_SMP */
79#ifdef CONFIG_SMP
80 return !cpumask_empty(&cpu_foreign_map[0]);
81#else
82 return false;
83#endif
84}
85
86/*
87 * Special Variant of smp_call_function for use by cache functions:
88 *
89 * o No return value
90 * o collapses to normal function call on UP kernels
91 * o collapses to normal function call on systems with a single shared
92 * primary cache.
93 * o doesn't disable interrupts on the local CPU
94 */
95static inline void r4k_on_each_cpu(unsigned int type,
96 void (*func)(void *info), void *info)
97{
98 preempt_disable();
99 if (r4k_op_needs_ipi(type))
100 smp_call_function_many(&cpu_foreign_map[smp_processor_id()],
101 func, info, 1);
102 func(info);
103 preempt_enable();
104}
105
106/*
107 * Must die.
108 */
109static unsigned long icache_size __read_mostly;
110static unsigned long dcache_size __read_mostly;
111static unsigned long vcache_size __read_mostly;
112static unsigned long scache_size __read_mostly;
113
114/*
115 * Dummy cache handling routines for machines without boardcaches
116 */
117static void cache_noop(void) {}
118
119static struct bcache_ops no_sc_ops = {
120 .bc_enable = (void *)cache_noop,
121 .bc_disable = (void *)cache_noop,
122 .bc_wback_inv = (void *)cache_noop,
123 .bc_inv = (void *)cache_noop
124};
125
126struct bcache_ops *bcops = &no_sc_ops;
127
128#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
129#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
130
131#define R4600_HIT_CACHEOP_WAR_IMPL \
132do { \
133 if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) && \
134 cpu_is_r4600_v2_x()) \
135 *(volatile unsigned long *)CKSEG1; \
136 if (IS_ENABLED(CONFIG_WAR_R4600_V1_HIT_CACHEOP)) \
137 __asm__ __volatile__("nop;nop;nop;nop"); \
138} while (0)
139
140static void (*r4k_blast_dcache_page)(unsigned long addr);
141
142static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
143{
144 R4600_HIT_CACHEOP_WAR_IMPL;
145 blast_dcache32_page(addr);
146}
147
148static inline void r4k_blast_dcache_page_dc64(unsigned long addr)
149{
150 blast_dcache64_page(addr);
151}
152
153static inline void r4k_blast_dcache_page_dc128(unsigned long addr)
154{
155 blast_dcache128_page(addr);
156}
157
158static void r4k_blast_dcache_page_setup(void)
159{
160 unsigned long dc_lsize = cpu_dcache_line_size();
161
162 switch (dc_lsize) {
163 case 0:
164 r4k_blast_dcache_page = (void *)cache_noop;
165 break;
166 case 16:
167 r4k_blast_dcache_page = blast_dcache16_page;
168 break;
169 case 32:
170 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
171 break;
172 case 64:
173 r4k_blast_dcache_page = r4k_blast_dcache_page_dc64;
174 break;
175 case 128:
176 r4k_blast_dcache_page = r4k_blast_dcache_page_dc128;
177 break;
178 default:
179 break;
180 }
181}
182
183#ifndef CONFIG_EVA
184#define r4k_blast_dcache_user_page r4k_blast_dcache_page
185#else
186
187static void (*r4k_blast_dcache_user_page)(unsigned long addr);
188
189static void r4k_blast_dcache_user_page_setup(void)
190{
191 unsigned long dc_lsize = cpu_dcache_line_size();
192
193 if (dc_lsize == 0)
194 r4k_blast_dcache_user_page = (void *)cache_noop;
195 else if (dc_lsize == 16)
196 r4k_blast_dcache_user_page = blast_dcache16_user_page;
197 else if (dc_lsize == 32)
198 r4k_blast_dcache_user_page = blast_dcache32_user_page;
199 else if (dc_lsize == 64)
200 r4k_blast_dcache_user_page = blast_dcache64_user_page;
201}
202
203#endif
204
205static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
206
207static void r4k_blast_dcache_page_indexed_setup(void)
208{
209 unsigned long dc_lsize = cpu_dcache_line_size();
210
211 if (dc_lsize == 0)
212 r4k_blast_dcache_page_indexed = (void *)cache_noop;
213 else if (dc_lsize == 16)
214 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
215 else if (dc_lsize == 32)
216 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
217 else if (dc_lsize == 64)
218 r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed;
219 else if (dc_lsize == 128)
220 r4k_blast_dcache_page_indexed = blast_dcache128_page_indexed;
221}
222
223void (* r4k_blast_dcache)(void);
224EXPORT_SYMBOL(r4k_blast_dcache);
225
226static void r4k_blast_dcache_setup(void)
227{
228 unsigned long dc_lsize = cpu_dcache_line_size();
229
230 if (dc_lsize == 0)
231 r4k_blast_dcache = (void *)cache_noop;
232 else if (dc_lsize == 16)
233 r4k_blast_dcache = blast_dcache16;
234 else if (dc_lsize == 32)
235 r4k_blast_dcache = blast_dcache32;
236 else if (dc_lsize == 64)
237 r4k_blast_dcache = blast_dcache64;
238 else if (dc_lsize == 128)
239 r4k_blast_dcache = blast_dcache128;
240}
241
242/* force code alignment (used for CONFIG_WAR_TX49XX_ICACHE_INDEX_INV) */
243#define JUMP_TO_ALIGN(order) \
244 __asm__ __volatile__( \
245 "b\t1f\n\t" \
246 ".align\t" #order "\n\t" \
247 "1:\n\t" \
248 )
249#define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */
250#define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11)
251
252static inline void blast_r4600_v1_icache32(void)
253{
254 unsigned long flags;
255
256 local_irq_save(flags);
257 blast_icache32();
258 local_irq_restore(flags);
259}
260
261static inline void tx49_blast_icache32(void)
262{
263 unsigned long start = INDEX_BASE;
264 unsigned long end = start + current_cpu_data.icache.waysize;
265 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
266 unsigned long ws_end = current_cpu_data.icache.ways <<
267 current_cpu_data.icache.waybit;
268 unsigned long ws, addr;
269
270 CACHE32_UNROLL32_ALIGN2;
271 /* I'm in even chunk. blast odd chunks */
272 for (ws = 0; ws < ws_end; ws += ws_inc)
273 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
274 cache_unroll(32, kernel_cache, Index_Invalidate_I,
275 addr | ws, 32);
276 CACHE32_UNROLL32_ALIGN;
277 /* I'm in odd chunk. blast even chunks */
278 for (ws = 0; ws < ws_end; ws += ws_inc)
279 for (addr = start; addr < end; addr += 0x400 * 2)
280 cache_unroll(32, kernel_cache, Index_Invalidate_I,
281 addr | ws, 32);
282}
283
284static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
285{
286 unsigned long flags;
287
288 local_irq_save(flags);
289 blast_icache32_page_indexed(page);
290 local_irq_restore(flags);
291}
292
293static inline void tx49_blast_icache32_page_indexed(unsigned long page)
294{
295 unsigned long indexmask = current_cpu_data.icache.waysize - 1;
296 unsigned long start = INDEX_BASE + (page & indexmask);
297 unsigned long end = start + PAGE_SIZE;
298 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
299 unsigned long ws_end = current_cpu_data.icache.ways <<
300 current_cpu_data.icache.waybit;
301 unsigned long ws, addr;
302
303 CACHE32_UNROLL32_ALIGN2;
304 /* I'm in even chunk. blast odd chunks */
305 for (ws = 0; ws < ws_end; ws += ws_inc)
306 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
307 cache_unroll(32, kernel_cache, Index_Invalidate_I,
308 addr | ws, 32);
309 CACHE32_UNROLL32_ALIGN;
310 /* I'm in odd chunk. blast even chunks */
311 for (ws = 0; ws < ws_end; ws += ws_inc)
312 for (addr = start; addr < end; addr += 0x400 * 2)
313 cache_unroll(32, kernel_cache, Index_Invalidate_I,
314 addr | ws, 32);
315}
316
317static void (* r4k_blast_icache_page)(unsigned long addr);
318
319static void r4k_blast_icache_page_setup(void)
320{
321 unsigned long ic_lsize = cpu_icache_line_size();
322
323 if (ic_lsize == 0)
324 r4k_blast_icache_page = (void *)cache_noop;
325 else if (ic_lsize == 16)
326 r4k_blast_icache_page = blast_icache16_page;
327 else if (ic_lsize == 32 && current_cpu_type() == CPU_LOONGSON2EF)
328 r4k_blast_icache_page = loongson2_blast_icache32_page;
329 else if (ic_lsize == 32)
330 r4k_blast_icache_page = blast_icache32_page;
331 else if (ic_lsize == 64)
332 r4k_blast_icache_page = blast_icache64_page;
333 else if (ic_lsize == 128)
334 r4k_blast_icache_page = blast_icache128_page;
335}
336
337#ifndef CONFIG_EVA
338#define r4k_blast_icache_user_page r4k_blast_icache_page
339#else
340
341static void (*r4k_blast_icache_user_page)(unsigned long addr);
342
343static void r4k_blast_icache_user_page_setup(void)
344{
345 unsigned long ic_lsize = cpu_icache_line_size();
346
347 if (ic_lsize == 0)
348 r4k_blast_icache_user_page = (void *)cache_noop;
349 else if (ic_lsize == 16)
350 r4k_blast_icache_user_page = blast_icache16_user_page;
351 else if (ic_lsize == 32)
352 r4k_blast_icache_user_page = blast_icache32_user_page;
353 else if (ic_lsize == 64)
354 r4k_blast_icache_user_page = blast_icache64_user_page;
355}
356
357#endif
358
359static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
360
361static void r4k_blast_icache_page_indexed_setup(void)
362{
363 unsigned long ic_lsize = cpu_icache_line_size();
364
365 if (ic_lsize == 0)
366 r4k_blast_icache_page_indexed = (void *)cache_noop;
367 else if (ic_lsize == 16)
368 r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
369 else if (ic_lsize == 32) {
370 if (IS_ENABLED(CONFIG_WAR_R4600_V1_INDEX_ICACHEOP) &&
371 cpu_is_r4600_v1_x())
372 r4k_blast_icache_page_indexed =
373 blast_icache32_r4600_v1_page_indexed;
374 else if (IS_ENABLED(CONFIG_WAR_TX49XX_ICACHE_INDEX_INV))
375 r4k_blast_icache_page_indexed =
376 tx49_blast_icache32_page_indexed;
377 else if (current_cpu_type() == CPU_LOONGSON2EF)
378 r4k_blast_icache_page_indexed =
379 loongson2_blast_icache32_page_indexed;
380 else
381 r4k_blast_icache_page_indexed =
382 blast_icache32_page_indexed;
383 } else if (ic_lsize == 64)
384 r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
385}
386
387void (* r4k_blast_icache)(void);
388EXPORT_SYMBOL(r4k_blast_icache);
389
390static void r4k_blast_icache_setup(void)
391{
392 unsigned long ic_lsize = cpu_icache_line_size();
393
394 if (ic_lsize == 0)
395 r4k_blast_icache = (void *)cache_noop;
396 else if (ic_lsize == 16)
397 r4k_blast_icache = blast_icache16;
398 else if (ic_lsize == 32) {
399 if (IS_ENABLED(CONFIG_WAR_R4600_V1_INDEX_ICACHEOP) &&
400 cpu_is_r4600_v1_x())
401 r4k_blast_icache = blast_r4600_v1_icache32;
402 else if (IS_ENABLED(CONFIG_WAR_TX49XX_ICACHE_INDEX_INV))
403 r4k_blast_icache = tx49_blast_icache32;
404 else if (current_cpu_type() == CPU_LOONGSON2EF)
405 r4k_blast_icache = loongson2_blast_icache32;
406 else
407 r4k_blast_icache = blast_icache32;
408 } else if (ic_lsize == 64)
409 r4k_blast_icache = blast_icache64;
410 else if (ic_lsize == 128)
411 r4k_blast_icache = blast_icache128;
412}
413
414static void (* r4k_blast_scache_page)(unsigned long addr);
415
416static void r4k_blast_scache_page_setup(void)
417{
418 unsigned long sc_lsize = cpu_scache_line_size();
419
420 if (scache_size == 0)
421 r4k_blast_scache_page = (void *)cache_noop;
422 else if (sc_lsize == 16)
423 r4k_blast_scache_page = blast_scache16_page;
424 else if (sc_lsize == 32)
425 r4k_blast_scache_page = blast_scache32_page;
426 else if (sc_lsize == 64)
427 r4k_blast_scache_page = blast_scache64_page;
428 else if (sc_lsize == 128)
429 r4k_blast_scache_page = blast_scache128_page;
430}
431
432static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
433
434static void r4k_blast_scache_page_indexed_setup(void)
435{
436 unsigned long sc_lsize = cpu_scache_line_size();
437
438 if (scache_size == 0)
439 r4k_blast_scache_page_indexed = (void *)cache_noop;
440 else if (sc_lsize == 16)
441 r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
442 else if (sc_lsize == 32)
443 r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
444 else if (sc_lsize == 64)
445 r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
446 else if (sc_lsize == 128)
447 r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
448}
449
450static void (* r4k_blast_scache)(void);
451
452static void r4k_blast_scache_setup(void)
453{
454 unsigned long sc_lsize = cpu_scache_line_size();
455
456 if (scache_size == 0)
457 r4k_blast_scache = (void *)cache_noop;
458 else if (sc_lsize == 16)
459 r4k_blast_scache = blast_scache16;
460 else if (sc_lsize == 32)
461 r4k_blast_scache = blast_scache32;
462 else if (sc_lsize == 64)
463 r4k_blast_scache = blast_scache64;
464 else if (sc_lsize == 128)
465 r4k_blast_scache = blast_scache128;
466}
467
468static void (*r4k_blast_scache_node)(long node);
469
470static void r4k_blast_scache_node_setup(void)
471{
472 unsigned long sc_lsize = cpu_scache_line_size();
473
474 if (current_cpu_type() != CPU_LOONGSON64)
475 r4k_blast_scache_node = (void *)cache_noop;
476 else if (sc_lsize == 16)
477 r4k_blast_scache_node = blast_scache16_node;
478 else if (sc_lsize == 32)
479 r4k_blast_scache_node = blast_scache32_node;
480 else if (sc_lsize == 64)
481 r4k_blast_scache_node = blast_scache64_node;
482 else if (sc_lsize == 128)
483 r4k_blast_scache_node = blast_scache128_node;
484}
485
486static inline void local_r4k___flush_cache_all(void * args)
487{
488 switch (current_cpu_type()) {
489 case CPU_LOONGSON2EF:
490 case CPU_R4000SC:
491 case CPU_R4000MC:
492 case CPU_R4400SC:
493 case CPU_R4400MC:
494 case CPU_R10000:
495 case CPU_R12000:
496 case CPU_R14000:
497 case CPU_R16000:
498 /*
499 * These caches are inclusive caches, that is, if something
500 * is not cached in the S-cache, we know it also won't be
501 * in one of the primary caches.
502 */
503 r4k_blast_scache();
504 break;
505
506 case CPU_LOONGSON64:
507 /* Use get_ebase_cpunum() for both NUMA=y/n */
508 r4k_blast_scache_node(get_ebase_cpunum() >> 2);
509 break;
510
511 case CPU_BMIPS5000:
512 r4k_blast_scache();
513 __sync();
514 break;
515
516 default:
517 r4k_blast_dcache();
518 r4k_blast_icache();
519 break;
520 }
521}
522
523static void r4k___flush_cache_all(void)
524{
525 r4k_on_each_cpu(R4K_INDEX, local_r4k___flush_cache_all, NULL);
526}
527
528/**
529 * has_valid_asid() - Determine if an mm already has an ASID.
530 * @mm: Memory map.
531 * @type: R4K_HIT or R4K_INDEX, type of cache op.
532 *
533 * Determines whether @mm already has an ASID on any of the CPUs which cache ops
534 * of type @type within an r4k_on_each_cpu() call will affect. If
535 * r4k_on_each_cpu() does an SMP call to a single VPE in each core, then the
536 * scope of the operation is confined to sibling CPUs, otherwise all online CPUs
537 * will need to be checked.
538 *
539 * Must be called in non-preemptive context.
540 *
541 * Returns: 1 if the CPUs affected by @type cache ops have an ASID for @mm.
542 * 0 otherwise.
543 */
544static inline int has_valid_asid(const struct mm_struct *mm, unsigned int type)
545{
546 unsigned int i;
547 const cpumask_t *mask = cpu_present_mask;
548
549 if (cpu_has_mmid)
550 return cpu_context(0, mm) != 0;
551
552 /* cpu_sibling_map[] undeclared when !CONFIG_SMP */
553#ifdef CONFIG_SMP
554 /*
555 * If r4k_on_each_cpu does SMP calls, it does them to a single VPE in
556 * each foreign core, so we only need to worry about siblings.
557 * Otherwise we need to worry about all present CPUs.
558 */
559 if (r4k_op_needs_ipi(type))
560 mask = &cpu_sibling_map[smp_processor_id()];
561#endif
562 for_each_cpu(i, mask)
563 if (cpu_context(i, mm))
564 return 1;
565 return 0;
566}
567
568static void r4k__flush_cache_vmap(void)
569{
570 r4k_blast_dcache();
571}
572
573static void r4k__flush_cache_vunmap(void)
574{
575 r4k_blast_dcache();
576}
577
578/*
579 * Note: flush_tlb_range() assumes flush_cache_range() sufficiently flushes
580 * whole caches when vma is executable.
581 */
582static inline void local_r4k_flush_cache_range(void * args)
583{
584 struct vm_area_struct *vma = args;
585 int exec = vma->vm_flags & VM_EXEC;
586
587 if (!has_valid_asid(vma->vm_mm, R4K_INDEX))
588 return;
589
590 /*
591 * If dcache can alias, we must blast it since mapping is changing.
592 * If executable, we must ensure any dirty lines are written back far
593 * enough to be visible to icache.
594 */
595 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc))
596 r4k_blast_dcache();
597 /* If executable, blast stale lines from icache */
598 if (exec)
599 r4k_blast_icache();
600}
601
602static void r4k_flush_cache_range(struct vm_area_struct *vma,
603 unsigned long start, unsigned long end)
604{
605 int exec = vma->vm_flags & VM_EXEC;
606
607 if (cpu_has_dc_aliases || exec)
608 r4k_on_each_cpu(R4K_INDEX, local_r4k_flush_cache_range, vma);
609}
610
611static inline void local_r4k_flush_cache_mm(void * args)
612{
613 struct mm_struct *mm = args;
614
615 if (!has_valid_asid(mm, R4K_INDEX))
616 return;
617
618 /*
619 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we
620 * only flush the primary caches but R1x000 behave sane ...
621 * R4000SC and R4400SC indexed S-cache ops also invalidate primary
622 * caches, so we can bail out early.
623 */
624 if (current_cpu_type() == CPU_R4000SC ||
625 current_cpu_type() == CPU_R4000MC ||
626 current_cpu_type() == CPU_R4400SC ||
627 current_cpu_type() == CPU_R4400MC) {
628 r4k_blast_scache();
629 return;
630 }
631
632 r4k_blast_dcache();
633}
634
635static void r4k_flush_cache_mm(struct mm_struct *mm)
636{
637 if (!cpu_has_dc_aliases)
638 return;
639
640 r4k_on_each_cpu(R4K_INDEX, local_r4k_flush_cache_mm, mm);
641}
642
643struct flush_cache_page_args {
644 struct vm_area_struct *vma;
645 unsigned long addr;
646 unsigned long pfn;
647};
648
649static inline void local_r4k_flush_cache_page(void *args)
650{
651 struct flush_cache_page_args *fcp_args = args;
652 struct vm_area_struct *vma = fcp_args->vma;
653 unsigned long addr = fcp_args->addr;
654 struct page *page = pfn_to_page(fcp_args->pfn);
655 int exec = vma->vm_flags & VM_EXEC;
656 struct mm_struct *mm = vma->vm_mm;
657 int map_coherent = 0;
658 pmd_t *pmdp;
659 pte_t *ptep;
660 void *vaddr;
661
662 /*
663 * If owns no valid ASID yet, cannot possibly have gotten
664 * this page into the cache.
665 */
666 if (!has_valid_asid(mm, R4K_HIT))
667 return;
668
669 addr &= PAGE_MASK;
670 pmdp = pmd_off(mm, addr);
671 ptep = pte_offset_kernel(pmdp, addr);
672
673 /*
674 * If the page isn't marked valid, the page cannot possibly be
675 * in the cache.
676 */
677 if (!(pte_present(*ptep)))
678 return;
679
680 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID))
681 vaddr = NULL;
682 else {
683 /*
684 * Use kmap_coherent or kmap_atomic to do flushes for
685 * another ASID than the current one.
686 */
687 map_coherent = (cpu_has_dc_aliases &&
688 page_mapcount(page) &&
689 !Page_dcache_dirty(page));
690 if (map_coherent)
691 vaddr = kmap_coherent(page, addr);
692 else
693 vaddr = kmap_atomic(page);
694 addr = (unsigned long)vaddr;
695 }
696
697 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
698 vaddr ? r4k_blast_dcache_page(addr) :
699 r4k_blast_dcache_user_page(addr);
700 if (exec && !cpu_icache_snoops_remote_store)
701 r4k_blast_scache_page(addr);
702 }
703 if (exec) {
704 if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) {
705 drop_mmu_context(mm);
706 } else
707 vaddr ? r4k_blast_icache_page(addr) :
708 r4k_blast_icache_user_page(addr);
709 }
710
711 if (vaddr) {
712 if (map_coherent)
713 kunmap_coherent();
714 else
715 kunmap_atomic(vaddr);
716 }
717}
718
719static void r4k_flush_cache_page(struct vm_area_struct *vma,
720 unsigned long addr, unsigned long pfn)
721{
722 struct flush_cache_page_args args;
723
724 args.vma = vma;
725 args.addr = addr;
726 args.pfn = pfn;
727
728 r4k_on_each_cpu(R4K_HIT, local_r4k_flush_cache_page, &args);
729}
730
731static inline void local_r4k_flush_data_cache_page(void * addr)
732{
733 r4k_blast_dcache_page((unsigned long) addr);
734}
735
736static void r4k_flush_data_cache_page(unsigned long addr)
737{
738 if (in_atomic())
739 local_r4k_flush_data_cache_page((void *)addr);
740 else
741 r4k_on_each_cpu(R4K_HIT, local_r4k_flush_data_cache_page,
742 (void *) addr);
743}
744
745struct flush_icache_range_args {
746 unsigned long start;
747 unsigned long end;
748 unsigned int type;
749 bool user;
750};
751
752static inline void __local_r4k_flush_icache_range(unsigned long start,
753 unsigned long end,
754 unsigned int type,
755 bool user)
756{
757 if (!cpu_has_ic_fills_f_dc) {
758 if (type == R4K_INDEX ||
759 (type & R4K_INDEX && end - start >= dcache_size)) {
760 r4k_blast_dcache();
761 } else {
762 R4600_HIT_CACHEOP_WAR_IMPL;
763 if (user)
764 protected_blast_dcache_range(start, end);
765 else
766 blast_dcache_range(start, end);
767 }
768 }
769
770 if (type == R4K_INDEX ||
771 (type & R4K_INDEX && end - start > icache_size))
772 r4k_blast_icache();
773 else {
774 switch (boot_cpu_type()) {
775 case CPU_LOONGSON2EF:
776 protected_loongson2_blast_icache_range(start, end);
777 break;
778
779 default:
780 if (user)
781 protected_blast_icache_range(start, end);
782 else
783 blast_icache_range(start, end);
784 break;
785 }
786 }
787}
788
789static inline void local_r4k_flush_icache_range(unsigned long start,
790 unsigned long end)
791{
792 __local_r4k_flush_icache_range(start, end, R4K_HIT | R4K_INDEX, false);
793}
794
795static inline void local_r4k_flush_icache_user_range(unsigned long start,
796 unsigned long end)
797{
798 __local_r4k_flush_icache_range(start, end, R4K_HIT | R4K_INDEX, true);
799}
800
801static inline void local_r4k_flush_icache_range_ipi(void *args)
802{
803 struct flush_icache_range_args *fir_args = args;
804 unsigned long start = fir_args->start;
805 unsigned long end = fir_args->end;
806 unsigned int type = fir_args->type;
807 bool user = fir_args->user;
808
809 __local_r4k_flush_icache_range(start, end, type, user);
810}
811
812static void __r4k_flush_icache_range(unsigned long start, unsigned long end,
813 bool user)
814{
815 struct flush_icache_range_args args;
816 unsigned long size, cache_size;
817
818 args.start = start;
819 args.end = end;
820 args.type = R4K_HIT | R4K_INDEX;
821 args.user = user;
822
823 /*
824 * Indexed cache ops require an SMP call.
825 * Consider if that can or should be avoided.
826 */
827 preempt_disable();
828 if (r4k_op_needs_ipi(R4K_INDEX) && !r4k_op_needs_ipi(R4K_HIT)) {
829 /*
830 * If address-based cache ops don't require an SMP call, then
831 * use them exclusively for small flushes.
832 */
833 size = end - start;
834 cache_size = icache_size;
835 if (!cpu_has_ic_fills_f_dc) {
836 size *= 2;
837 cache_size += dcache_size;
838 }
839 if (size <= cache_size)
840 args.type &= ~R4K_INDEX;
841 }
842 r4k_on_each_cpu(args.type, local_r4k_flush_icache_range_ipi, &args);
843 preempt_enable();
844 instruction_hazard();
845}
846
847static void r4k_flush_icache_range(unsigned long start, unsigned long end)
848{
849 return __r4k_flush_icache_range(start, end, false);
850}
851
852static void r4k_flush_icache_user_range(unsigned long start, unsigned long end)
853{
854 return __r4k_flush_icache_range(start, end, true);
855}
856
857#ifdef CONFIG_DMA_NONCOHERENT
858
859static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
860{
861 /* Catch bad driver code */
862 if (WARN_ON(size == 0))
863 return;
864
865 preempt_disable();
866 if (cpu_has_inclusive_pcaches) {
867 if (size >= scache_size) {
868 if (current_cpu_type() != CPU_LOONGSON64)
869 r4k_blast_scache();
870 else
871 r4k_blast_scache_node(pa_to_nid(addr));
872 } else {
873 blast_scache_range(addr, addr + size);
874 }
875 preempt_enable();
876 __sync();
877 return;
878 }
879
880 /*
881 * Either no secondary cache or the available caches don't have the
882 * subset property so we have to flush the primary caches
883 * explicitly.
884 * If we would need IPI to perform an INDEX-type operation, then
885 * we have to use the HIT-type alternative as IPI cannot be used
886 * here due to interrupts possibly being disabled.
887 */
888 if (!r4k_op_needs_ipi(R4K_INDEX) && size >= dcache_size) {
889 r4k_blast_dcache();
890 } else {
891 R4600_HIT_CACHEOP_WAR_IMPL;
892 blast_dcache_range(addr, addr + size);
893 }
894 preempt_enable();
895
896 bc_wback_inv(addr, size);
897 __sync();
898}
899
900static void prefetch_cache_inv(unsigned long addr, unsigned long size)
901{
902 unsigned int linesz = cpu_scache_line_size();
903 unsigned long addr0 = addr, addr1;
904
905 addr0 &= ~(linesz - 1);
906 addr1 = (addr0 + size - 1) & ~(linesz - 1);
907
908 protected_writeback_scache_line(addr0);
909 if (likely(addr1 != addr0))
910 protected_writeback_scache_line(addr1);
911 else
912 return;
913
914 addr0 += linesz;
915 if (likely(addr1 != addr0))
916 protected_writeback_scache_line(addr0);
917 else
918 return;
919
920 addr1 -= linesz;
921 if (likely(addr1 > addr0))
922 protected_writeback_scache_line(addr0);
923}
924
925static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
926{
927 /* Catch bad driver code */
928 if (WARN_ON(size == 0))
929 return;
930
931 preempt_disable();
932
933 if (current_cpu_type() == CPU_BMIPS5000)
934 prefetch_cache_inv(addr, size);
935
936 if (cpu_has_inclusive_pcaches) {
937 if (size >= scache_size) {
938 if (current_cpu_type() != CPU_LOONGSON64)
939 r4k_blast_scache();
940 else
941 r4k_blast_scache_node(pa_to_nid(addr));
942 } else {
943 /*
944 * There is no clearly documented alignment requirement
945 * for the cache instruction on MIPS processors and
946 * some processors, among them the RM5200 and RM7000
947 * QED processors will throw an address error for cache
948 * hit ops with insufficient alignment. Solved by
949 * aligning the address to cache line size.
950 */
951 blast_inv_scache_range(addr, addr + size);
952 }
953 preempt_enable();
954 __sync();
955 return;
956 }
957
958 if (!r4k_op_needs_ipi(R4K_INDEX) && size >= dcache_size) {
959 r4k_blast_dcache();
960 } else {
961 R4600_HIT_CACHEOP_WAR_IMPL;
962 blast_inv_dcache_range(addr, addr + size);
963 }
964 preempt_enable();
965
966 bc_inv(addr, size);
967 __sync();
968}
969#endif /* CONFIG_DMA_NONCOHERENT */
970
971static void r4k_flush_icache_all(void)
972{
973 if (cpu_has_vtag_icache)
974 r4k_blast_icache();
975}
976
977struct flush_kernel_vmap_range_args {
978 unsigned long vaddr;
979 int size;
980};
981
982static inline void local_r4k_flush_kernel_vmap_range_index(void *args)
983{
984 /*
985 * Aliases only affect the primary caches so don't bother with
986 * S-caches or T-caches.
987 */
988 r4k_blast_dcache();
989}
990
991static inline void local_r4k_flush_kernel_vmap_range(void *args)
992{
993 struct flush_kernel_vmap_range_args *vmra = args;
994 unsigned long vaddr = vmra->vaddr;
995 int size = vmra->size;
996
997 /*
998 * Aliases only affect the primary caches so don't bother with
999 * S-caches or T-caches.
1000 */
1001 R4600_HIT_CACHEOP_WAR_IMPL;
1002 blast_dcache_range(vaddr, vaddr + size);
1003}
1004
1005static void r4k_flush_kernel_vmap_range(unsigned long vaddr, int size)
1006{
1007 struct flush_kernel_vmap_range_args args;
1008
1009 args.vaddr = (unsigned long) vaddr;
1010 args.size = size;
1011
1012 if (size >= dcache_size)
1013 r4k_on_each_cpu(R4K_INDEX,
1014 local_r4k_flush_kernel_vmap_range_index, NULL);
1015 else
1016 r4k_on_each_cpu(R4K_HIT, local_r4k_flush_kernel_vmap_range,
1017 &args);
1018}
1019
1020static inline void rm7k_erratum31(void)
1021{
1022 const unsigned long ic_lsize = 32;
1023 unsigned long addr;
1024
1025 /* RM7000 erratum #31. The icache is screwed at startup. */
1026 write_c0_taglo(0);
1027 write_c0_taghi(0);
1028
1029 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
1030 __asm__ __volatile__ (
1031 ".set push\n\t"
1032 ".set noreorder\n\t"
1033 ".set mips3\n\t"
1034 "cache\t%1, 0(%0)\n\t"
1035 "cache\t%1, 0x1000(%0)\n\t"
1036 "cache\t%1, 0x2000(%0)\n\t"
1037 "cache\t%1, 0x3000(%0)\n\t"
1038 "cache\t%2, 0(%0)\n\t"
1039 "cache\t%2, 0x1000(%0)\n\t"
1040 "cache\t%2, 0x2000(%0)\n\t"
1041 "cache\t%2, 0x3000(%0)\n\t"
1042 "cache\t%1, 0(%0)\n\t"
1043 "cache\t%1, 0x1000(%0)\n\t"
1044 "cache\t%1, 0x2000(%0)\n\t"
1045 "cache\t%1, 0x3000(%0)\n\t"
1046 ".set pop\n"
1047 :
1048 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill_I));
1049 }
1050}
1051
1052static inline int alias_74k_erratum(struct cpuinfo_mips *c)
1053{
1054 unsigned int imp = c->processor_id & PRID_IMP_MASK;
1055 unsigned int rev = c->processor_id & PRID_REV_MASK;
1056 int present = 0;
1057
1058 /*
1059 * Early versions of the 74K do not update the cache tags on a
1060 * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG
1061 * aliases. In this case it is better to treat the cache as always
1062 * having aliases. Also disable the synonym tag update feature
1063 * where available. In this case no opportunistic tag update will
1064 * happen where a load causes a virtual address miss but a physical
1065 * address hit during a D-cache look-up.
1066 */
1067 switch (imp) {
1068 case PRID_IMP_74K:
1069 if (rev <= PRID_REV_ENCODE_332(2, 4, 0))
1070 present = 1;
1071 if (rev == PRID_REV_ENCODE_332(2, 4, 0))
1072 write_c0_config6(read_c0_config6() | MTI_CONF6_SYND);
1073 break;
1074 case PRID_IMP_1074K:
1075 if (rev <= PRID_REV_ENCODE_332(1, 1, 0)) {
1076 present = 1;
1077 write_c0_config6(read_c0_config6() | MTI_CONF6_SYND);
1078 }
1079 break;
1080 default:
1081 BUG();
1082 }
1083
1084 return present;
1085}
1086
1087static void b5k_instruction_hazard(void)
1088{
1089 __sync();
1090 __sync();
1091 __asm__ __volatile__(
1092 " nop; nop; nop; nop; nop; nop; nop; nop\n"
1093 " nop; nop; nop; nop; nop; nop; nop; nop\n"
1094 " nop; nop; nop; nop; nop; nop; nop; nop\n"
1095 " nop; nop; nop; nop; nop; nop; nop; nop\n"
1096 : : : "memory");
1097}
1098
1099static char *way_string[] = { NULL, "direct mapped", "2-way",
1100 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way",
1101 "9-way", "10-way", "11-way", "12-way",
1102 "13-way", "14-way", "15-way", "16-way",
1103};
1104
1105static void probe_pcache(void)
1106{
1107 struct cpuinfo_mips *c = &current_cpu_data;
1108 unsigned int config = read_c0_config();
1109 unsigned int prid = read_c0_prid();
1110 int has_74k_erratum = 0;
1111 unsigned long config1;
1112 unsigned int lsize;
1113
1114 switch (current_cpu_type()) {
1115 case CPU_R4600: /* QED style two way caches? */
1116 case CPU_R4700:
1117 case CPU_R5000:
1118 case CPU_NEVADA:
1119 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1120 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1121 c->icache.ways = 2;
1122 c->icache.waybit = __ffs(icache_size/2);
1123
1124 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1125 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1126 c->dcache.ways = 2;
1127 c->dcache.waybit= __ffs(dcache_size/2);
1128
1129 c->options |= MIPS_CPU_CACHE_CDEX_P;
1130 break;
1131
1132 case CPU_R5500:
1133 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1134 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1135 c->icache.ways = 2;
1136 c->icache.waybit= 0;
1137
1138 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1139 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1140 c->dcache.ways = 2;
1141 c->dcache.waybit = 0;
1142
1143 c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH;
1144 break;
1145
1146 case CPU_TX49XX:
1147 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1148 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1149 c->icache.ways = 4;
1150 c->icache.waybit= 0;
1151
1152 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1153 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1154 c->dcache.ways = 4;
1155 c->dcache.waybit = 0;
1156
1157 c->options |= MIPS_CPU_CACHE_CDEX_P;
1158 c->options |= MIPS_CPU_PREFETCH;
1159 break;
1160
1161 case CPU_R4000PC:
1162 case CPU_R4000SC:
1163 case CPU_R4000MC:
1164 case CPU_R4400PC:
1165 case CPU_R4400SC:
1166 case CPU_R4400MC:
1167 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1168 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1169 c->icache.ways = 1;
1170 c->icache.waybit = 0; /* doesn't matter */
1171
1172 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1173 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1174 c->dcache.ways = 1;
1175 c->dcache.waybit = 0; /* does not matter */
1176
1177 c->options |= MIPS_CPU_CACHE_CDEX_P;
1178 break;
1179
1180 case CPU_R10000:
1181 case CPU_R12000:
1182 case CPU_R14000:
1183 case CPU_R16000:
1184 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29));
1185 c->icache.linesz = 64;
1186 c->icache.ways = 2;
1187 c->icache.waybit = 0;
1188
1189 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26));
1190 c->dcache.linesz = 32;
1191 c->dcache.ways = 2;
1192 c->dcache.waybit = 0;
1193
1194 c->options |= MIPS_CPU_PREFETCH;
1195 break;
1196
1197 case CPU_VR4133:
1198 write_c0_config(config & ~VR41_CONF_P4K);
1199 fallthrough;
1200 case CPU_VR4131:
1201 /* Workaround for cache instruction bug of VR4131 */
1202 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U ||
1203 c->processor_id == 0x0c82U) {
1204 config |= 0x00400000U;
1205 if (c->processor_id == 0x0c80U)
1206 config |= VR41_CONF_BP;
1207 write_c0_config(config);
1208 } else
1209 c->options |= MIPS_CPU_CACHE_CDEX_P;
1210
1211 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
1212 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1213 c->icache.ways = 2;
1214 c->icache.waybit = __ffs(icache_size/2);
1215
1216 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
1217 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1218 c->dcache.ways = 2;
1219 c->dcache.waybit = __ffs(dcache_size/2);
1220 break;
1221
1222 case CPU_VR41XX:
1223 case CPU_VR4111:
1224 case CPU_VR4121:
1225 case CPU_VR4122:
1226 case CPU_VR4181:
1227 case CPU_VR4181A:
1228 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
1229 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1230 c->icache.ways = 1;
1231 c->icache.waybit = 0; /* doesn't matter */
1232
1233 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
1234 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1235 c->dcache.ways = 1;
1236 c->dcache.waybit = 0; /* does not matter */
1237
1238 c->options |= MIPS_CPU_CACHE_CDEX_P;
1239 break;
1240
1241 case CPU_RM7000:
1242 rm7k_erratum31();
1243
1244 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1245 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1246 c->icache.ways = 4;
1247 c->icache.waybit = __ffs(icache_size / c->icache.ways);
1248
1249 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1250 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1251 c->dcache.ways = 4;
1252 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways);
1253
1254 c->options |= MIPS_CPU_CACHE_CDEX_P;
1255 c->options |= MIPS_CPU_PREFETCH;
1256 break;
1257
1258 case CPU_LOONGSON2EF:
1259 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1260 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1261 if (prid & 0x3)
1262 c->icache.ways = 4;
1263 else
1264 c->icache.ways = 2;
1265 c->icache.waybit = 0;
1266
1267 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1268 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1269 if (prid & 0x3)
1270 c->dcache.ways = 4;
1271 else
1272 c->dcache.ways = 2;
1273 c->dcache.waybit = 0;
1274 break;
1275
1276 case CPU_LOONGSON64:
1277 config1 = read_c0_config1();
1278 lsize = (config1 >> 19) & 7;
1279 if (lsize)
1280 c->icache.linesz = 2 << lsize;
1281 else
1282 c->icache.linesz = 0;
1283 c->icache.sets = 64 << ((config1 >> 22) & 7);
1284 c->icache.ways = 1 + ((config1 >> 16) & 7);
1285 icache_size = c->icache.sets *
1286 c->icache.ways *
1287 c->icache.linesz;
1288 c->icache.waybit = 0;
1289
1290 lsize = (config1 >> 10) & 7;
1291 if (lsize)
1292 c->dcache.linesz = 2 << lsize;
1293 else
1294 c->dcache.linesz = 0;
1295 c->dcache.sets = 64 << ((config1 >> 13) & 7);
1296 c->dcache.ways = 1 + ((config1 >> 7) & 7);
1297 dcache_size = c->dcache.sets *
1298 c->dcache.ways *
1299 c->dcache.linesz;
1300 c->dcache.waybit = 0;
1301 if ((c->processor_id & (PRID_IMP_MASK | PRID_REV_MASK)) >=
1302 (PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0) ||
1303 (c->processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64R)
1304 c->options |= MIPS_CPU_PREFETCH;
1305 break;
1306
1307 case CPU_CAVIUM_OCTEON3:
1308 /* For now lie about the number of ways. */
1309 c->icache.linesz = 128;
1310 c->icache.sets = 16;
1311 c->icache.ways = 8;
1312 c->icache.flags |= MIPS_CACHE_VTAG;
1313 icache_size = c->icache.sets * c->icache.ways * c->icache.linesz;
1314
1315 c->dcache.linesz = 128;
1316 c->dcache.ways = 8;
1317 c->dcache.sets = 8;
1318 dcache_size = c->dcache.sets * c->dcache.ways * c->dcache.linesz;
1319 c->options |= MIPS_CPU_PREFETCH;
1320 break;
1321
1322 default:
1323 if (!(config & MIPS_CONF_M))
1324 panic("Don't know how to probe P-caches on this cpu.");
1325
1326 /*
1327 * So we seem to be a MIPS32 or MIPS64 CPU
1328 * So let's probe the I-cache ...
1329 */
1330 config1 = read_c0_config1();
1331
1332 lsize = (config1 >> 19) & 7;
1333
1334 /* IL == 7 is reserved */
1335 if (lsize == 7)
1336 panic("Invalid icache line size");
1337
1338 c->icache.linesz = lsize ? 2 << lsize : 0;
1339
1340 c->icache.sets = 32 << (((config1 >> 22) + 1) & 7);
1341 c->icache.ways = 1 + ((config1 >> 16) & 7);
1342
1343 icache_size = c->icache.sets *
1344 c->icache.ways *
1345 c->icache.linesz;
1346 c->icache.waybit = __ffs(icache_size/c->icache.ways);
1347
1348 if (config & MIPS_CONF_VI)
1349 c->icache.flags |= MIPS_CACHE_VTAG;
1350
1351 /*
1352 * Now probe the MIPS32 / MIPS64 data cache.
1353 */
1354 c->dcache.flags = 0;
1355
1356 lsize = (config1 >> 10) & 7;
1357
1358 /* DL == 7 is reserved */
1359 if (lsize == 7)
1360 panic("Invalid dcache line size");
1361
1362 c->dcache.linesz = lsize ? 2 << lsize : 0;
1363
1364 c->dcache.sets = 32 << (((config1 >> 13) + 1) & 7);
1365 c->dcache.ways = 1 + ((config1 >> 7) & 7);
1366
1367 dcache_size = c->dcache.sets *
1368 c->dcache.ways *
1369 c->dcache.linesz;
1370 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways);
1371
1372 c->options |= MIPS_CPU_PREFETCH;
1373 break;
1374 }
1375
1376 /*
1377 * Processor configuration sanity check for the R4000SC erratum
1378 * #5. With page sizes larger than 32kB there is no possibility
1379 * to get a VCE exception anymore so we don't care about this
1380 * misconfiguration. The case is rather theoretical anyway;
1381 * presumably no vendor is shipping his hardware in the "bad"
1382 * configuration.
1383 */
1384 if ((prid & PRID_IMP_MASK) == PRID_IMP_R4000 &&
1385 (prid & PRID_REV_MASK) < PRID_REV_R4400 &&
1386 !(config & CONF_SC) && c->icache.linesz != 16 &&
1387 PAGE_SIZE <= 0x8000)
1388 panic("Improper R4000SC processor configuration detected");
1389
1390 /* compute a couple of other cache variables */
1391 c->icache.waysize = icache_size / c->icache.ways;
1392 c->dcache.waysize = dcache_size / c->dcache.ways;
1393
1394 c->icache.sets = c->icache.linesz ?
1395 icache_size / (c->icache.linesz * c->icache.ways) : 0;
1396 c->dcache.sets = c->dcache.linesz ?
1397 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0;
1398
1399 /*
1400 * R1x000 P-caches are odd in a positive way. They're 32kB 2-way
1401 * virtually indexed so normally would suffer from aliases. So
1402 * normally they'd suffer from aliases but magic in the hardware deals
1403 * with that for us so we don't need to take care ourselves.
1404 */
1405 switch (current_cpu_type()) {
1406 case CPU_20KC:
1407 case CPU_25KF:
1408 case CPU_I6400:
1409 case CPU_I6500:
1410 case CPU_SB1:
1411 case CPU_SB1A:
1412 case CPU_XLR:
1413 c->dcache.flags |= MIPS_CACHE_PINDEX;
1414 break;
1415
1416 case CPU_R10000:
1417 case CPU_R12000:
1418 case CPU_R14000:
1419 case CPU_R16000:
1420 break;
1421
1422 case CPU_74K:
1423 case CPU_1074K:
1424 has_74k_erratum = alias_74k_erratum(c);
1425 fallthrough;
1426 case CPU_M14KC:
1427 case CPU_M14KEC:
1428 case CPU_24K:
1429 case CPU_34K:
1430 case CPU_1004K:
1431 case CPU_INTERAPTIV:
1432 case CPU_P5600:
1433 case CPU_PROAPTIV:
1434 case CPU_M5150:
1435 case CPU_QEMU_GENERIC:
1436 case CPU_P6600:
1437 case CPU_M6250:
1438 if (!(read_c0_config7() & MIPS_CONF7_IAR) &&
1439 (c->icache.waysize > PAGE_SIZE))
1440 c->icache.flags |= MIPS_CACHE_ALIASES;
1441 if (!has_74k_erratum && (read_c0_config7() & MIPS_CONF7_AR)) {
1442 /*
1443 * Effectively physically indexed dcache,
1444 * thus no virtual aliases.
1445 */
1446 c->dcache.flags |= MIPS_CACHE_PINDEX;
1447 break;
1448 }
1449 fallthrough;
1450 default:
1451 if (has_74k_erratum || c->dcache.waysize > PAGE_SIZE)
1452 c->dcache.flags |= MIPS_CACHE_ALIASES;
1453 }
1454
1455 /* Physically indexed caches don't suffer from virtual aliasing */
1456 if (c->dcache.flags & MIPS_CACHE_PINDEX)
1457 c->dcache.flags &= ~MIPS_CACHE_ALIASES;
1458
1459 /*
1460 * In systems with CM the icache fills from L2 or closer caches, and
1461 * thus sees remote stores without needing to write them back any
1462 * further than that.
1463 */
1464 if (mips_cm_present())
1465 c->icache.flags |= MIPS_IC_SNOOPS_REMOTE;
1466
1467 switch (current_cpu_type()) {
1468 case CPU_20KC:
1469 /*
1470 * Some older 20Kc chips doesn't have the 'VI' bit in
1471 * the config register.
1472 */
1473 c->icache.flags |= MIPS_CACHE_VTAG;
1474 break;
1475
1476 case CPU_ALCHEMY:
1477 case CPU_I6400:
1478 case CPU_I6500:
1479 c->icache.flags |= MIPS_CACHE_IC_F_DC;
1480 break;
1481
1482 case CPU_BMIPS5000:
1483 c->icache.flags |= MIPS_CACHE_IC_F_DC;
1484 /* Cache aliases are handled in hardware; allow HIGHMEM */
1485 c->dcache.flags &= ~MIPS_CACHE_ALIASES;
1486 break;
1487
1488 case CPU_LOONGSON2EF:
1489 /*
1490 * LOONGSON2 has 4 way icache, but when using indexed cache op,
1491 * one op will act on all 4 ways
1492 */
1493 c->icache.ways = 1;
1494 }
1495
1496 pr_info("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
1497 icache_size >> 10,
1498 c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT",
1499 way_string[c->icache.ways], c->icache.linesz);
1500
1501 pr_info("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n",
1502 dcache_size >> 10, way_string[c->dcache.ways],
1503 (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT",
1504 (c->dcache.flags & MIPS_CACHE_ALIASES) ?
1505 "cache aliases" : "no aliases",
1506 c->dcache.linesz);
1507}
1508
1509static void probe_vcache(void)
1510{
1511 struct cpuinfo_mips *c = &current_cpu_data;
1512 unsigned int config2, lsize;
1513
1514 if (current_cpu_type() != CPU_LOONGSON64)
1515 return;
1516
1517 config2 = read_c0_config2();
1518 if ((lsize = ((config2 >> 20) & 15)))
1519 c->vcache.linesz = 2 << lsize;
1520 else
1521 c->vcache.linesz = lsize;
1522
1523 c->vcache.sets = 64 << ((config2 >> 24) & 15);
1524 c->vcache.ways = 1 + ((config2 >> 16) & 15);
1525
1526 vcache_size = c->vcache.sets * c->vcache.ways * c->vcache.linesz;
1527
1528 c->vcache.waybit = 0;
1529 c->vcache.waysize = vcache_size / c->vcache.ways;
1530
1531 pr_info("Unified victim cache %ldkB %s, linesize %d bytes.\n",
1532 vcache_size >> 10, way_string[c->vcache.ways], c->vcache.linesz);
1533}
1534
1535/*
1536 * If you even _breathe_ on this function, look at the gcc output and make sure
1537 * it does not pop things on and off the stack for the cache sizing loop that
1538 * executes in KSEG1 space or else you will crash and burn badly. You have
1539 * been warned.
1540 */
1541static int probe_scache(void)
1542{
1543 unsigned long flags, addr, begin, end, pow2;
1544 unsigned int config = read_c0_config();
1545 struct cpuinfo_mips *c = &current_cpu_data;
1546
1547 if (config & CONF_SC)
1548 return 0;
1549
1550 begin = (unsigned long) &_stext;
1551 begin &= ~((4 * 1024 * 1024) - 1);
1552 end = begin + (4 * 1024 * 1024);
1553
1554 /*
1555 * This is such a bitch, you'd think they would make it easy to do
1556 * this. Away you daemons of stupidity!
1557 */
1558 local_irq_save(flags);
1559
1560 /* Fill each size-multiple cache line with a valid tag. */
1561 pow2 = (64 * 1024);
1562 for (addr = begin; addr < end; addr = (begin + pow2)) {
1563 unsigned long *p = (unsigned long *) addr;
1564 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
1565 pow2 <<= 1;
1566 }
1567
1568 /* Load first line with zero (therefore invalid) tag. */
1569 write_c0_taglo(0);
1570 write_c0_taghi(0);
1571 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
1572 cache_op(Index_Store_Tag_I, begin);
1573 cache_op(Index_Store_Tag_D, begin);
1574 cache_op(Index_Store_Tag_SD, begin);
1575
1576 /* Now search for the wrap around point. */
1577 pow2 = (128 * 1024);
1578 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) {
1579 cache_op(Index_Load_Tag_SD, addr);
1580 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
1581 if (!read_c0_taglo())
1582 break;
1583 pow2 <<= 1;
1584 }
1585 local_irq_restore(flags);
1586 addr -= begin;
1587
1588 scache_size = addr;
1589 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22);
1590 c->scache.ways = 1;
1591 c->scache.waybit = 0; /* does not matter */
1592
1593 return 1;
1594}
1595
1596static void loongson2_sc_init(void)
1597{
1598 struct cpuinfo_mips *c = &current_cpu_data;
1599
1600 scache_size = 512*1024;
1601 c->scache.linesz = 32;
1602 c->scache.ways = 4;
1603 c->scache.waybit = 0;
1604 c->scache.waysize = scache_size / (c->scache.ways);
1605 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1606 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1607 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1608
1609 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1610}
1611
1612static void loongson3_sc_init(void)
1613{
1614 struct cpuinfo_mips *c = &current_cpu_data;
1615 unsigned int config2, lsize;
1616
1617 config2 = read_c0_config2();
1618 lsize = (config2 >> 4) & 15;
1619 if (lsize)
1620 c->scache.linesz = 2 << lsize;
1621 else
1622 c->scache.linesz = 0;
1623 c->scache.sets = 64 << ((config2 >> 8) & 15);
1624 c->scache.ways = 1 + (config2 & 15);
1625
1626 scache_size = c->scache.sets *
1627 c->scache.ways *
1628 c->scache.linesz;
1629
1630 /* Loongson-3 has 4-Scache banks, while Loongson-2K have only 2 banks */
1631 if ((c->processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64R)
1632 scache_size *= 2;
1633 else
1634 scache_size *= 4;
1635
1636 c->scache.waybit = 0;
1637 c->scache.waysize = scache_size / c->scache.ways;
1638 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1639 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1640 if (scache_size)
1641 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1642 return;
1643}
1644
1645extern int r5k_sc_init(void);
1646extern int rm7k_sc_init(void);
1647extern int mips_sc_init(void);
1648
1649static void setup_scache(void)
1650{
1651 struct cpuinfo_mips *c = &current_cpu_data;
1652 unsigned int config = read_c0_config();
1653 int sc_present = 0;
1654
1655 /*
1656 * Do the probing thing on R4000SC and R4400SC processors. Other
1657 * processors don't have a S-cache that would be relevant to the
1658 * Linux memory management.
1659 */
1660 switch (current_cpu_type()) {
1661 case CPU_R4000SC:
1662 case CPU_R4000MC:
1663 case CPU_R4400SC:
1664 case CPU_R4400MC:
1665 sc_present = run_uncached(probe_scache);
1666 if (sc_present)
1667 c->options |= MIPS_CPU_CACHE_CDEX_S;
1668 break;
1669
1670 case CPU_R10000:
1671 case CPU_R12000:
1672 case CPU_R14000:
1673 case CPU_R16000:
1674 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16);
1675 c->scache.linesz = 64 << ((config >> 13) & 1);
1676 c->scache.ways = 2;
1677 c->scache.waybit= 0;
1678 sc_present = 1;
1679 break;
1680
1681 case CPU_R5000:
1682 case CPU_NEVADA:
1683#ifdef CONFIG_R5000_CPU_SCACHE
1684 r5k_sc_init();
1685#endif
1686 return;
1687
1688 case CPU_RM7000:
1689#ifdef CONFIG_RM7000_CPU_SCACHE
1690 rm7k_sc_init();
1691#endif
1692 return;
1693
1694 case CPU_LOONGSON2EF:
1695 loongson2_sc_init();
1696 return;
1697
1698 case CPU_LOONGSON64:
1699 loongson3_sc_init();
1700 return;
1701
1702 case CPU_CAVIUM_OCTEON3:
1703 case CPU_XLP:
1704 /* don't need to worry about L2, fully coherent */
1705 return;
1706
1707 default:
1708 if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 |
1709 MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 |
1710 MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 |
1711 MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) {
1712#ifdef CONFIG_MIPS_CPU_SCACHE
1713 if (mips_sc_init ()) {
1714 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz;
1715 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n",
1716 scache_size >> 10,
1717 way_string[c->scache.ways], c->scache.linesz);
1718
1719 if (current_cpu_type() == CPU_BMIPS5000)
1720 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1721 }
1722
1723#else
1724 if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT))
1725 panic("Dunno how to handle MIPS32 / MIPS64 second level cache");
1726#endif
1727 return;
1728 }
1729 sc_present = 0;
1730 }
1731
1732 if (!sc_present)
1733 return;
1734
1735 /* compute a couple of other cache variables */
1736 c->scache.waysize = scache_size / c->scache.ways;
1737
1738 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1739
1740 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1741 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1742
1743 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1744}
1745
1746void au1x00_fixup_config_od(void)
1747{
1748 /*
1749 * c0_config.od (bit 19) was write only (and read as 0)
1750 * on the early revisions of Alchemy SOCs. It disables the bus
1751 * transaction overlapping and needs to be set to fix various errata.
1752 */
1753 switch (read_c0_prid()) {
1754 case 0x00030100: /* Au1000 DA */
1755 case 0x00030201: /* Au1000 HA */
1756 case 0x00030202: /* Au1000 HB */
1757 case 0x01030200: /* Au1500 AB */
1758 /*
1759 * Au1100 errata actually keeps silence about this bit, so we set it
1760 * just in case for those revisions that require it to be set according
1761 * to the (now gone) cpu table.
1762 */
1763 case 0x02030200: /* Au1100 AB */
1764 case 0x02030201: /* Au1100 BA */
1765 case 0x02030202: /* Au1100 BC */
1766 set_c0_config(1 << 19);
1767 break;
1768 }
1769}
1770
1771/* CP0 hazard avoidance. */
1772#define NXP_BARRIER() \
1773 __asm__ __volatile__( \
1774 ".set noreorder\n\t" \
1775 "nop; nop; nop; nop; nop; nop;\n\t" \
1776 ".set reorder\n\t")
1777
1778static void nxp_pr4450_fixup_config(void)
1779{
1780 unsigned long config0;
1781
1782 config0 = read_c0_config();
1783
1784 /* clear all three cache coherency fields */
1785 config0 &= ~(0x7 | (7 << 25) | (7 << 28));
1786 config0 |= (((_page_cachable_default >> _CACHE_SHIFT) << 0) |
1787 ((_page_cachable_default >> _CACHE_SHIFT) << 25) |
1788 ((_page_cachable_default >> _CACHE_SHIFT) << 28));
1789 write_c0_config(config0);
1790 NXP_BARRIER();
1791}
1792
1793static int cca = -1;
1794
1795static int __init cca_setup(char *str)
1796{
1797 get_option(&str, &cca);
1798
1799 return 0;
1800}
1801
1802early_param("cca", cca_setup);
1803
1804static void coherency_setup(void)
1805{
1806 if (cca < 0 || cca > 7)
1807 cca = read_c0_config() & CONF_CM_CMASK;
1808 _page_cachable_default = cca << _CACHE_SHIFT;
1809
1810 pr_debug("Using cache attribute %d\n", cca);
1811 change_c0_config(CONF_CM_CMASK, cca);
1812
1813 /*
1814 * c0_status.cu=0 specifies that updates by the sc instruction use
1815 * the coherency mode specified by the TLB; 1 means cachable
1816 * coherent update on write will be used. Not all processors have
1817 * this bit and; some wire it to zero, others like Toshiba had the
1818 * silly idea of putting something else there ...
1819 */
1820 switch (current_cpu_type()) {
1821 case CPU_R4000PC:
1822 case CPU_R4000SC:
1823 case CPU_R4000MC:
1824 case CPU_R4400PC:
1825 case CPU_R4400SC:
1826 case CPU_R4400MC:
1827 clear_c0_config(CONF_CU);
1828 break;
1829 /*
1830 * We need to catch the early Alchemy SOCs with
1831 * the write-only co_config.od bit and set it back to one on:
1832 * Au1000 rev DA, HA, HB; Au1100 AB, BA, BC, Au1500 AB
1833 */
1834 case CPU_ALCHEMY:
1835 au1x00_fixup_config_od();
1836 break;
1837
1838 case PRID_IMP_PR4450:
1839 nxp_pr4450_fixup_config();
1840 break;
1841 }
1842}
1843
1844static void r4k_cache_error_setup(void)
1845{
1846 extern char __weak except_vec2_generic;
1847 extern char __weak except_vec2_sb1;
1848
1849 switch (current_cpu_type()) {
1850 case CPU_SB1:
1851 case CPU_SB1A:
1852 set_uncached_handler(0x100, &except_vec2_sb1, 0x80);
1853 break;
1854
1855 default:
1856 set_uncached_handler(0x100, &except_vec2_generic, 0x80);
1857 break;
1858 }
1859}
1860
1861void r4k_cache_init(void)
1862{
1863 extern void build_clear_page(void);
1864 extern void build_copy_page(void);
1865 struct cpuinfo_mips *c = &current_cpu_data;
1866
1867 probe_pcache();
1868 probe_vcache();
1869 setup_scache();
1870
1871 r4k_blast_dcache_page_setup();
1872 r4k_blast_dcache_page_indexed_setup();
1873 r4k_blast_dcache_setup();
1874 r4k_blast_icache_page_setup();
1875 r4k_blast_icache_page_indexed_setup();
1876 r4k_blast_icache_setup();
1877 r4k_blast_scache_page_setup();
1878 r4k_blast_scache_page_indexed_setup();
1879 r4k_blast_scache_setup();
1880 r4k_blast_scache_node_setup();
1881#ifdef CONFIG_EVA
1882 r4k_blast_dcache_user_page_setup();
1883 r4k_blast_icache_user_page_setup();
1884#endif
1885
1886 /*
1887 * Some MIPS32 and MIPS64 processors have physically indexed caches.
1888 * This code supports virtually indexed processors and will be
1889 * unnecessarily inefficient on physically indexed processors.
1890 */
1891 if (c->dcache.linesz && cpu_has_dc_aliases)
1892 shm_align_mask = max_t( unsigned long,
1893 c->dcache.sets * c->dcache.linesz - 1,
1894 PAGE_SIZE - 1);
1895 else
1896 shm_align_mask = PAGE_SIZE-1;
1897
1898 __flush_cache_vmap = r4k__flush_cache_vmap;
1899 __flush_cache_vunmap = r4k__flush_cache_vunmap;
1900
1901 flush_cache_all = cache_noop;
1902 __flush_cache_all = r4k___flush_cache_all;
1903 flush_cache_mm = r4k_flush_cache_mm;
1904 flush_cache_page = r4k_flush_cache_page;
1905 flush_cache_range = r4k_flush_cache_range;
1906
1907 __flush_kernel_vmap_range = r4k_flush_kernel_vmap_range;
1908
1909 flush_icache_all = r4k_flush_icache_all;
1910 local_flush_data_cache_page = local_r4k_flush_data_cache_page;
1911 flush_data_cache_page = r4k_flush_data_cache_page;
1912 flush_icache_range = r4k_flush_icache_range;
1913 local_flush_icache_range = local_r4k_flush_icache_range;
1914 __flush_icache_user_range = r4k_flush_icache_user_range;
1915 __local_flush_icache_user_range = local_r4k_flush_icache_user_range;
1916
1917#ifdef CONFIG_DMA_NONCOHERENT
1918#ifdef CONFIG_DMA_MAYBE_COHERENT
1919 if (coherentio == IO_COHERENCE_ENABLED ||
1920 (coherentio == IO_COHERENCE_DEFAULT && hw_coherentio)) {
1921 _dma_cache_wback_inv = (void *)cache_noop;
1922 _dma_cache_wback = (void *)cache_noop;
1923 _dma_cache_inv = (void *)cache_noop;
1924 } else
1925#endif /* CONFIG_DMA_MAYBE_COHERENT */
1926 {
1927 _dma_cache_wback_inv = r4k_dma_cache_wback_inv;
1928 _dma_cache_wback = r4k_dma_cache_wback_inv;
1929 _dma_cache_inv = r4k_dma_cache_inv;
1930 }
1931#endif /* CONFIG_DMA_NONCOHERENT */
1932
1933 build_clear_page();
1934 build_copy_page();
1935
1936 /*
1937 * We want to run CMP kernels on core with and without coherent
1938 * caches. Therefore, do not use CONFIG_MIPS_CMP to decide whether
1939 * or not to flush caches.
1940 */
1941 local_r4k___flush_cache_all(NULL);
1942
1943 coherency_setup();
1944 board_cache_error_setup = r4k_cache_error_setup;
1945
1946 /*
1947 * Per-CPU overrides
1948 */
1949 switch (current_cpu_type()) {
1950 case CPU_BMIPS4350:
1951 case CPU_BMIPS4380:
1952 /* No IPI is needed because all CPUs share the same D$ */
1953 flush_data_cache_page = r4k_blast_dcache_page;
1954 break;
1955 case CPU_BMIPS5000:
1956 /* We lose our superpowers if L2 is disabled */
1957 if (c->scache.flags & MIPS_CACHE_NOT_PRESENT)
1958 break;
1959
1960 /* I$ fills from D$ just by emptying the write buffers */
1961 flush_cache_page = (void *)b5k_instruction_hazard;
1962 flush_cache_range = (void *)b5k_instruction_hazard;
1963 local_flush_data_cache_page = (void *)b5k_instruction_hazard;
1964 flush_data_cache_page = (void *)b5k_instruction_hazard;
1965 flush_icache_range = (void *)b5k_instruction_hazard;
1966 local_flush_icache_range = (void *)b5k_instruction_hazard;
1967
1968
1969 /* Optimization: an L2 flush implicitly flushes the L1 */
1970 current_cpu_data.options |= MIPS_CPU_INCLUSIVE_CACHES;
1971 break;
1972 case CPU_LOONGSON64:
1973 /* Loongson-3 maintains cache coherency by hardware */
1974 __flush_cache_all = cache_noop;
1975 __flush_cache_vmap = cache_noop;
1976 __flush_cache_vunmap = cache_noop;
1977 __flush_kernel_vmap_range = (void *)cache_noop;
1978 flush_cache_mm = (void *)cache_noop;
1979 flush_cache_page = (void *)cache_noop;
1980 flush_cache_range = (void *)cache_noop;
1981 flush_icache_all = (void *)cache_noop;
1982 flush_data_cache_page = (void *)cache_noop;
1983 local_flush_data_cache_page = (void *)cache_noop;
1984 break;
1985 }
1986}
1987
1988static int r4k_cache_pm_notifier(struct notifier_block *self, unsigned long cmd,
1989 void *v)
1990{
1991 switch (cmd) {
1992 case CPU_PM_ENTER_FAILED:
1993 case CPU_PM_EXIT:
1994 coherency_setup();
1995 break;
1996 }
1997
1998 return NOTIFY_OK;
1999}
2000
2001static struct notifier_block r4k_cache_pm_notifier_block = {
2002 .notifier_call = r4k_cache_pm_notifier,
2003};
2004
2005int __init r4k_cache_init_pm(void)
2006{
2007 return cpu_pm_register_notifier(&r4k_cache_pm_notifier_block);
2008}
2009arch_initcall(r4k_cache_init_pm);
diff --git a/arch/mips/mm/c-tx39.c b/arch/mips/mm/c-tx39.c
new file mode 100644
index 000000000..03dfbb40e
--- /dev/null
+++ b/arch/mips/mm/c-tx39.c
@@ -0,0 +1,414 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * r2300.c: R2000 and R3000 specific mmu/cache code.
4 *
5 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
6 *
7 * with a lot of changes to make this thing work for R3000s
8 * Tx39XX R4k style caches added. HK
9 * Copyright (C) 1998, 1999, 2000 Harald Koerfgen
10 * Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
11 */
12#include <linux/init.h>
13#include <linux/kernel.h>
14#include <linux/sched.h>
15#include <linux/smp.h>
16#include <linux/mm.h>
17
18#include <asm/cacheops.h>
19#include <asm/page.h>
20#include <asm/mmu_context.h>
21#include <asm/isadep.h>
22#include <asm/io.h>
23#include <asm/bootinfo.h>
24#include <asm/cpu.h>
25
26/* For R3000 cores with R4000 style caches */
27static unsigned long icache_size, dcache_size; /* Size in bytes */
28
29#include <asm/r4kcache.h>
30
31/* This sequence is required to ensure icache is disabled immediately */
32#define TX39_STOP_STREAMING() \
33__asm__ __volatile__( \
34 ".set push\n\t" \
35 ".set noreorder\n\t" \
36 "b 1f\n\t" \
37 "nop\n\t" \
38 "1:\n\t" \
39 ".set pop" \
40 )
41
42/* TX39H-style cache flush routines. */
43static void tx39h_flush_icache_all(void)
44{
45 unsigned long flags, config;
46
47 /* disable icache (set ICE#) */
48 local_irq_save(flags);
49 config = read_c0_conf();
50 write_c0_conf(config & ~TX39_CONF_ICE);
51 TX39_STOP_STREAMING();
52 blast_icache16();
53 write_c0_conf(config);
54 local_irq_restore(flags);
55}
56
57static void tx39h_dma_cache_wback_inv(unsigned long addr, unsigned long size)
58{
59 /* Catch bad driver code */
60 BUG_ON(size == 0);
61
62 iob();
63 blast_inv_dcache_range(addr, addr + size);
64}
65
66
67/* TX39H2,TX39H3 */
68static inline void tx39_blast_dcache_page(unsigned long addr)
69{
70 if (current_cpu_type() != CPU_TX3912)
71 blast_dcache16_page(addr);
72}
73
74static inline void tx39_blast_dcache_page_indexed(unsigned long addr)
75{
76 blast_dcache16_page_indexed(addr);
77}
78
79static inline void tx39_blast_dcache(void)
80{
81 blast_dcache16();
82}
83
84static inline void tx39_blast_icache_page(unsigned long addr)
85{
86 unsigned long flags, config;
87 /* disable icache (set ICE#) */
88 local_irq_save(flags);
89 config = read_c0_conf();
90 write_c0_conf(config & ~TX39_CONF_ICE);
91 TX39_STOP_STREAMING();
92 blast_icache16_page(addr);
93 write_c0_conf(config);
94 local_irq_restore(flags);
95}
96
97static inline void tx39_blast_icache_page_indexed(unsigned long addr)
98{
99 unsigned long flags, config;
100 /* disable icache (set ICE#) */
101 local_irq_save(flags);
102 config = read_c0_conf();
103 write_c0_conf(config & ~TX39_CONF_ICE);
104 TX39_STOP_STREAMING();
105 blast_icache16_page_indexed(addr);
106 write_c0_conf(config);
107 local_irq_restore(flags);
108}
109
110static inline void tx39_blast_icache(void)
111{
112 unsigned long flags, config;
113 /* disable icache (set ICE#) */
114 local_irq_save(flags);
115 config = read_c0_conf();
116 write_c0_conf(config & ~TX39_CONF_ICE);
117 TX39_STOP_STREAMING();
118 blast_icache16();
119 write_c0_conf(config);
120 local_irq_restore(flags);
121}
122
123static void tx39__flush_cache_vmap(void)
124{
125 tx39_blast_dcache();
126}
127
128static void tx39__flush_cache_vunmap(void)
129{
130 tx39_blast_dcache();
131}
132
133static inline void tx39_flush_cache_all(void)
134{
135 if (!cpu_has_dc_aliases)
136 return;
137
138 tx39_blast_dcache();
139}
140
141static inline void tx39___flush_cache_all(void)
142{
143 tx39_blast_dcache();
144 tx39_blast_icache();
145}
146
147static void tx39_flush_cache_mm(struct mm_struct *mm)
148{
149 if (!cpu_has_dc_aliases)
150 return;
151
152 if (cpu_context(smp_processor_id(), mm) != 0)
153 tx39_blast_dcache();
154}
155
156static void tx39_flush_cache_range(struct vm_area_struct *vma,
157 unsigned long start, unsigned long end)
158{
159 if (!cpu_has_dc_aliases)
160 return;
161 if (!(cpu_context(smp_processor_id(), vma->vm_mm)))
162 return;
163
164 tx39_blast_dcache();
165}
166
167static void tx39_flush_cache_page(struct vm_area_struct *vma, unsigned long page, unsigned long pfn)
168{
169 int exec = vma->vm_flags & VM_EXEC;
170 struct mm_struct *mm = vma->vm_mm;
171 pmd_t *pmdp;
172 pte_t *ptep;
173
174 /*
175 * If ownes no valid ASID yet, cannot possibly have gotten
176 * this page into the cache.
177 */
178 if (cpu_context(smp_processor_id(), mm) == 0)
179 return;
180
181 page &= PAGE_MASK;
182 pmdp = pmd_off(mm, page);
183 ptep = pte_offset_kernel(pmdp, page);
184
185 /*
186 * If the page isn't marked valid, the page cannot possibly be
187 * in the cache.
188 */
189 if (!(pte_val(*ptep) & _PAGE_PRESENT))
190 return;
191
192 /*
193 * Doing flushes for another ASID than the current one is
194 * too difficult since stupid R4k caches do a TLB translation
195 * for every cache flush operation. So we do indexed flushes
196 * in that case, which doesn't overly flush the cache too much.
197 */
198 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) {
199 if (cpu_has_dc_aliases || exec)
200 tx39_blast_dcache_page(page);
201 if (exec)
202 tx39_blast_icache_page(page);
203
204 return;
205 }
206
207 /*
208 * Do indexed flush, too much work to get the (possible) TLB refills
209 * to work correctly.
210 */
211 if (cpu_has_dc_aliases || exec)
212 tx39_blast_dcache_page_indexed(page);
213 if (exec)
214 tx39_blast_icache_page_indexed(page);
215}
216
217static void local_tx39_flush_data_cache_page(void * addr)
218{
219 tx39_blast_dcache_page((unsigned long)addr);
220}
221
222static void tx39_flush_data_cache_page(unsigned long addr)
223{
224 tx39_blast_dcache_page(addr);
225}
226
227static void tx39_flush_icache_range(unsigned long start, unsigned long end)
228{
229 if (end - start > dcache_size)
230 tx39_blast_dcache();
231 else
232 protected_blast_dcache_range(start, end);
233
234 if (end - start > icache_size)
235 tx39_blast_icache();
236 else {
237 unsigned long flags, config;
238 /* disable icache (set ICE#) */
239 local_irq_save(flags);
240 config = read_c0_conf();
241 write_c0_conf(config & ~TX39_CONF_ICE);
242 TX39_STOP_STREAMING();
243 protected_blast_icache_range(start, end);
244 write_c0_conf(config);
245 local_irq_restore(flags);
246 }
247}
248
249static void tx39_flush_kernel_vmap_range(unsigned long vaddr, int size)
250{
251 BUG();
252}
253
254static void tx39_dma_cache_wback_inv(unsigned long addr, unsigned long size)
255{
256 unsigned long end;
257
258 if (((size | addr) & (PAGE_SIZE - 1)) == 0) {
259 end = addr + size;
260 do {
261 tx39_blast_dcache_page(addr);
262 addr += PAGE_SIZE;
263 } while(addr != end);
264 } else if (size > dcache_size) {
265 tx39_blast_dcache();
266 } else {
267 blast_dcache_range(addr, addr + size);
268 }
269}
270
271static void tx39_dma_cache_inv(unsigned long addr, unsigned long size)
272{
273 unsigned long end;
274
275 if (((size | addr) & (PAGE_SIZE - 1)) == 0) {
276 end = addr + size;
277 do {
278 tx39_blast_dcache_page(addr);
279 addr += PAGE_SIZE;
280 } while(addr != end);
281 } else if (size > dcache_size) {
282 tx39_blast_dcache();
283 } else {
284 blast_inv_dcache_range(addr, addr + size);
285 }
286}
287
288static __init void tx39_probe_cache(void)
289{
290 unsigned long config;
291
292 config = read_c0_conf();
293
294 icache_size = 1 << (10 + ((config & TX39_CONF_ICS_MASK) >>
295 TX39_CONF_ICS_SHIFT));
296 dcache_size = 1 << (10 + ((config & TX39_CONF_DCS_MASK) >>
297 TX39_CONF_DCS_SHIFT));
298
299 current_cpu_data.icache.linesz = 16;
300 switch (current_cpu_type()) {
301 case CPU_TX3912:
302 current_cpu_data.icache.ways = 1;
303 current_cpu_data.dcache.ways = 1;
304 current_cpu_data.dcache.linesz = 4;
305 break;
306
307 case CPU_TX3927:
308 current_cpu_data.icache.ways = 2;
309 current_cpu_data.dcache.ways = 2;
310 current_cpu_data.dcache.linesz = 16;
311 break;
312
313 case CPU_TX3922:
314 default:
315 current_cpu_data.icache.ways = 1;
316 current_cpu_data.dcache.ways = 1;
317 current_cpu_data.dcache.linesz = 16;
318 break;
319 }
320}
321
322void tx39_cache_init(void)
323{
324 extern void build_clear_page(void);
325 extern void build_copy_page(void);
326 unsigned long config;
327
328 config = read_c0_conf();
329 config &= ~TX39_CONF_WBON;
330 write_c0_conf(config);
331
332 tx39_probe_cache();
333
334 switch (current_cpu_type()) {
335 case CPU_TX3912:
336 /* TX39/H core (writethru direct-map cache) */
337 __flush_cache_vmap = tx39__flush_cache_vmap;
338 __flush_cache_vunmap = tx39__flush_cache_vunmap;
339 flush_cache_all = tx39h_flush_icache_all;
340 __flush_cache_all = tx39h_flush_icache_all;
341 flush_cache_mm = (void *) tx39h_flush_icache_all;
342 flush_cache_range = (void *) tx39h_flush_icache_all;
343 flush_cache_page = (void *) tx39h_flush_icache_all;
344 flush_icache_range = (void *) tx39h_flush_icache_all;
345 local_flush_icache_range = (void *) tx39h_flush_icache_all;
346
347 local_flush_data_cache_page = (void *) tx39h_flush_icache_all;
348 flush_data_cache_page = (void *) tx39h_flush_icache_all;
349
350 _dma_cache_wback_inv = tx39h_dma_cache_wback_inv;
351
352 shm_align_mask = PAGE_SIZE - 1;
353
354 break;
355
356 case CPU_TX3922:
357 case CPU_TX3927:
358 default:
359 /* TX39/H2,H3 core (writeback 2way-set-associative cache) */
360 /* board-dependent init code may set WBON */
361
362 __flush_cache_vmap = tx39__flush_cache_vmap;
363 __flush_cache_vunmap = tx39__flush_cache_vunmap;
364
365 flush_cache_all = tx39_flush_cache_all;
366 __flush_cache_all = tx39___flush_cache_all;
367 flush_cache_mm = tx39_flush_cache_mm;
368 flush_cache_range = tx39_flush_cache_range;
369 flush_cache_page = tx39_flush_cache_page;
370 flush_icache_range = tx39_flush_icache_range;
371 local_flush_icache_range = tx39_flush_icache_range;
372
373 __flush_kernel_vmap_range = tx39_flush_kernel_vmap_range;
374
375 local_flush_data_cache_page = local_tx39_flush_data_cache_page;
376 flush_data_cache_page = tx39_flush_data_cache_page;
377
378 _dma_cache_wback_inv = tx39_dma_cache_wback_inv;
379 _dma_cache_wback = tx39_dma_cache_wback_inv;
380 _dma_cache_inv = tx39_dma_cache_inv;
381
382 shm_align_mask = max_t(unsigned long,
383 (dcache_size / current_cpu_data.dcache.ways) - 1,
384 PAGE_SIZE - 1);
385
386 break;
387 }
388
389 __flush_icache_user_range = flush_icache_range;
390 __local_flush_icache_user_range = local_flush_icache_range;
391
392 current_cpu_data.icache.waysize = icache_size / current_cpu_data.icache.ways;
393 current_cpu_data.dcache.waysize = dcache_size / current_cpu_data.dcache.ways;
394
395 current_cpu_data.icache.sets =
396 current_cpu_data.icache.waysize / current_cpu_data.icache.linesz;
397 current_cpu_data.dcache.sets =
398 current_cpu_data.dcache.waysize / current_cpu_data.dcache.linesz;
399
400 if (current_cpu_data.dcache.waysize > PAGE_SIZE)
401 current_cpu_data.dcache.flags |= MIPS_CACHE_ALIASES;
402
403 current_cpu_data.icache.waybit = 0;
404 current_cpu_data.dcache.waybit = 0;
405
406 pr_info("Primary instruction cache %ldkB, linesize %d bytes\n",
407 icache_size >> 10, current_cpu_data.icache.linesz);
408 pr_info("Primary data cache %ldkB, linesize %d bytes\n",
409 dcache_size >> 10, current_cpu_data.dcache.linesz);
410
411 build_clear_page();
412 build_copy_page();
413 tx39h_flush_icache_all();
414}
diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c
new file mode 100644
index 000000000..3e81ba000
--- /dev/null
+++ b/arch/mips/mm/cache.c
@@ -0,0 +1,242 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (C) 2007 MIPS Technologies, Inc.
8 */
9#include <linux/fs.h>
10#include <linux/fcntl.h>
11#include <linux/kernel.h>
12#include <linux/linkage.h>
13#include <linux/export.h>
14#include <linux/sched.h>
15#include <linux/syscalls.h>
16#include <linux/mm.h>
17#include <linux/highmem.h>
18
19#include <asm/cacheflush.h>
20#include <asm/processor.h>
21#include <asm/cpu.h>
22#include <asm/cpu-features.h>
23#include <asm/setup.h>
24
25/* Cache operations. */
26void (*flush_cache_all)(void);
27void (*__flush_cache_all)(void);
28EXPORT_SYMBOL_GPL(__flush_cache_all);
29void (*flush_cache_mm)(struct mm_struct *mm);
30void (*flush_cache_range)(struct vm_area_struct *vma, unsigned long start,
31 unsigned long end);
32void (*flush_cache_page)(struct vm_area_struct *vma, unsigned long page,
33 unsigned long pfn);
34void (*flush_icache_range)(unsigned long start, unsigned long end);
35EXPORT_SYMBOL_GPL(flush_icache_range);
36void (*local_flush_icache_range)(unsigned long start, unsigned long end);
37EXPORT_SYMBOL_GPL(local_flush_icache_range);
38void (*__flush_icache_user_range)(unsigned long start, unsigned long end);
39void (*__local_flush_icache_user_range)(unsigned long start, unsigned long end);
40EXPORT_SYMBOL_GPL(__local_flush_icache_user_range);
41
42void (*__flush_cache_vmap)(void);
43void (*__flush_cache_vunmap)(void);
44
45void (*__flush_kernel_vmap_range)(unsigned long vaddr, int size);
46EXPORT_SYMBOL_GPL(__flush_kernel_vmap_range);
47
48/* MIPS specific cache operations */
49void (*local_flush_data_cache_page)(void * addr);
50void (*flush_data_cache_page)(unsigned long addr);
51void (*flush_icache_all)(void);
52
53EXPORT_SYMBOL_GPL(local_flush_data_cache_page);
54EXPORT_SYMBOL(flush_data_cache_page);
55EXPORT_SYMBOL(flush_icache_all);
56
57#ifdef CONFIG_DMA_NONCOHERENT
58
59/* DMA cache operations. */
60void (*_dma_cache_wback_inv)(unsigned long start, unsigned long size);
61void (*_dma_cache_wback)(unsigned long start, unsigned long size);
62void (*_dma_cache_inv)(unsigned long start, unsigned long size);
63
64#endif /* CONFIG_DMA_NONCOHERENT */
65
66/*
67 * We could optimize the case where the cache argument is not BCACHE but
68 * that seems very atypical use ...
69 */
70SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes,
71 unsigned int, cache)
72{
73 if (bytes == 0)
74 return 0;
75 if (!access_ok((void __user *) addr, bytes))
76 return -EFAULT;
77
78 __flush_icache_user_range(addr, addr + bytes);
79
80 return 0;
81}
82
83void __flush_dcache_page(struct page *page)
84{
85 struct address_space *mapping = page_mapping_file(page);
86 unsigned long addr;
87
88 if (mapping && !mapping_mapped(mapping)) {
89 SetPageDcacheDirty(page);
90 return;
91 }
92
93 /*
94 * We could delay the flush for the !page_mapping case too. But that
95 * case is for exec env/arg pages and those are %99 certainly going to
96 * get faulted into the tlb (and thus flushed) anyways.
97 */
98 if (PageHighMem(page))
99 addr = (unsigned long)kmap_atomic(page);
100 else
101 addr = (unsigned long)page_address(page);
102
103 flush_data_cache_page(addr);
104
105 if (PageHighMem(page))
106 kunmap_atomic((void *)addr);
107}
108
109EXPORT_SYMBOL(__flush_dcache_page);
110
111void __flush_anon_page(struct page *page, unsigned long vmaddr)
112{
113 unsigned long addr = (unsigned long) page_address(page);
114
115 if (pages_do_alias(addr, vmaddr)) {
116 if (page_mapcount(page) && !Page_dcache_dirty(page)) {
117 void *kaddr;
118
119 kaddr = kmap_coherent(page, vmaddr);
120 flush_data_cache_page((unsigned long)kaddr);
121 kunmap_coherent();
122 } else
123 flush_data_cache_page(addr);
124 }
125}
126
127EXPORT_SYMBOL(__flush_anon_page);
128
129void __update_cache(unsigned long address, pte_t pte)
130{
131 struct page *page;
132 unsigned long pfn, addr;
133 int exec = !pte_no_exec(pte) && !cpu_has_ic_fills_f_dc;
134
135 pfn = pte_pfn(pte);
136 if (unlikely(!pfn_valid(pfn)))
137 return;
138 page = pfn_to_page(pfn);
139 if (Page_dcache_dirty(page)) {
140 if (PageHighMem(page))
141 addr = (unsigned long)kmap_atomic(page);
142 else
143 addr = (unsigned long)page_address(page);
144
145 if (exec || pages_do_alias(addr, address & PAGE_MASK))
146 flush_data_cache_page(addr);
147
148 if (PageHighMem(page))
149 kunmap_atomic((void *)addr);
150
151 ClearPageDcacheDirty(page);
152 }
153}
154
155unsigned long _page_cachable_default;
156EXPORT_SYMBOL(_page_cachable_default);
157
158static inline void setup_protection_map(void)
159{
160 if (cpu_has_rixi) {
161 protection_map[0] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ);
162 protection_map[1] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC);
163 protection_map[2] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ);
164 protection_map[3] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC);
165 protection_map[4] = __pgprot(_page_cachable_default | _PAGE_PRESENT);
166 protection_map[5] = __pgprot(_page_cachable_default | _PAGE_PRESENT);
167 protection_map[6] = __pgprot(_page_cachable_default | _PAGE_PRESENT);
168 protection_map[7] = __pgprot(_page_cachable_default | _PAGE_PRESENT);
169
170 protection_map[8] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ);
171 protection_map[9] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC);
172 protection_map[10] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE | _PAGE_NO_READ);
173 protection_map[11] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE);
174 protection_map[12] = __pgprot(_page_cachable_default | _PAGE_PRESENT);
175 protection_map[13] = __pgprot(_page_cachable_default | _PAGE_PRESENT);
176 protection_map[14] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_WRITE);
177 protection_map[15] = __pgprot(_page_cachable_default | _PAGE_PRESENT | _PAGE_WRITE);
178
179 } else {
180 protection_map[0] = PAGE_NONE;
181 protection_map[1] = PAGE_READONLY;
182 protection_map[2] = PAGE_COPY;
183 protection_map[3] = PAGE_COPY;
184 protection_map[4] = PAGE_READONLY;
185 protection_map[5] = PAGE_READONLY;
186 protection_map[6] = PAGE_COPY;
187 protection_map[7] = PAGE_COPY;
188 protection_map[8] = PAGE_NONE;
189 protection_map[9] = PAGE_READONLY;
190 protection_map[10] = PAGE_SHARED;
191 protection_map[11] = PAGE_SHARED;
192 protection_map[12] = PAGE_READONLY;
193 protection_map[13] = PAGE_READONLY;
194 protection_map[14] = PAGE_SHARED;
195 protection_map[15] = PAGE_SHARED;
196 }
197}
198
199void cpu_cache_init(void)
200{
201 if (cpu_has_3k_cache) {
202 extern void __weak r3k_cache_init(void);
203
204 r3k_cache_init();
205 }
206 if (cpu_has_6k_cache) {
207 extern void __weak r6k_cache_init(void);
208
209 r6k_cache_init();
210 }
211 if (cpu_has_4k_cache) {
212 extern void __weak r4k_cache_init(void);
213
214 r4k_cache_init();
215 }
216 if (cpu_has_8k_cache) {
217 extern void __weak r8k_cache_init(void);
218
219 r8k_cache_init();
220 }
221 if (cpu_has_tx39_cache) {
222 extern void __weak tx39_cache_init(void);
223
224 tx39_cache_init();
225 }
226
227 if (cpu_has_octeon_cache) {
228 extern void __weak octeon_cache_init(void);
229
230 octeon_cache_init();
231 }
232
233 setup_protection_map();
234}
235
236int __weak __uncached_access(struct file *file, unsigned long addr)
237{
238 if (file->f_flags & O_DSYNC)
239 return 1;
240
241 return addr >= __pa(high_memory);
242}
diff --git a/arch/mips/mm/cerr-sb1.c b/arch/mips/mm/cerr-sb1.c
new file mode 100644
index 000000000..a3c02df19
--- /dev/null
+++ b/arch/mips/mm/cerr-sb1.c
@@ -0,0 +1,569 @@
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2001,2002,2003 Broadcom Corporation
4 */
5#include <linux/sched.h>
6#include <asm/mipsregs.h>
7#include <asm/sibyte/sb1250.h>
8#include <asm/sibyte/sb1250_regs.h>
9
10#if !defined(CONFIG_SIBYTE_BUS_WATCHER) || defined(CONFIG_SIBYTE_BW_TRACE)
11#include <asm/io.h>
12#include <asm/sibyte/sb1250_scd.h>
13#endif
14
15/*
16 * We'd like to dump the L2_ECC_TAG register on errors, but errata make
17 * that unsafe... So for now we don't. (BCM1250/BCM112x erratum SOC-48.)
18 */
19#undef DUMP_L2_ECC_TAG_ON_ERROR
20
21/* SB1 definitions */
22
23/* XXX should come from config1 XXX */
24#define SB1_CACHE_INDEX_MASK 0x1fe0
25
26#define CP0_ERRCTL_RECOVERABLE (1 << 31)
27#define CP0_ERRCTL_DCACHE (1 << 30)
28#define CP0_ERRCTL_ICACHE (1 << 29)
29#define CP0_ERRCTL_MULTIBUS (1 << 23)
30#define CP0_ERRCTL_MC_TLB (1 << 15)
31#define CP0_ERRCTL_MC_TIMEOUT (1 << 14)
32
33#define CP0_CERRI_TAG_PARITY (1 << 29)
34#define CP0_CERRI_DATA_PARITY (1 << 28)
35#define CP0_CERRI_EXTERNAL (1 << 26)
36
37#define CP0_CERRI_IDX_VALID(c) (!((c) & CP0_CERRI_EXTERNAL))
38#define CP0_CERRI_DATA (CP0_CERRI_DATA_PARITY)
39
40#define CP0_CERRD_MULTIPLE (1 << 31)
41#define CP0_CERRD_TAG_STATE (1 << 30)
42#define CP0_CERRD_TAG_ADDRESS (1 << 29)
43#define CP0_CERRD_DATA_SBE (1 << 28)
44#define CP0_CERRD_DATA_DBE (1 << 27)
45#define CP0_CERRD_EXTERNAL (1 << 26)
46#define CP0_CERRD_LOAD (1 << 25)
47#define CP0_CERRD_STORE (1 << 24)
48#define CP0_CERRD_FILLWB (1 << 23)
49#define CP0_CERRD_COHERENCY (1 << 22)
50#define CP0_CERRD_DUPTAG (1 << 21)
51
52#define CP0_CERRD_DPA_VALID(c) (!((c) & CP0_CERRD_EXTERNAL))
53#define CP0_CERRD_IDX_VALID(c) \
54 (((c) & (CP0_CERRD_LOAD | CP0_CERRD_STORE)) ? (!((c) & CP0_CERRD_EXTERNAL)) : 0)
55#define CP0_CERRD_CAUSES \
56 (CP0_CERRD_LOAD | CP0_CERRD_STORE | CP0_CERRD_FILLWB | CP0_CERRD_COHERENCY | CP0_CERRD_DUPTAG)
57#define CP0_CERRD_TYPES \
58 (CP0_CERRD_TAG_STATE | CP0_CERRD_TAG_ADDRESS | CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE | CP0_CERRD_EXTERNAL)
59#define CP0_CERRD_DATA (CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE)
60
61static uint32_t extract_ic(unsigned short addr, int data);
62static uint32_t extract_dc(unsigned short addr, int data);
63
64static inline void breakout_errctl(unsigned int val)
65{
66 if (val & CP0_ERRCTL_RECOVERABLE)
67 printk(" recoverable");
68 if (val & CP0_ERRCTL_DCACHE)
69 printk(" dcache");
70 if (val & CP0_ERRCTL_ICACHE)
71 printk(" icache");
72 if (val & CP0_ERRCTL_MULTIBUS)
73 printk(" multiple-buserr");
74 printk("\n");
75}
76
77static inline void breakout_cerri(unsigned int val)
78{
79 if (val & CP0_CERRI_TAG_PARITY)
80 printk(" tag-parity");
81 if (val & CP0_CERRI_DATA_PARITY)
82 printk(" data-parity");
83 if (val & CP0_CERRI_EXTERNAL)
84 printk(" external");
85 printk("\n");
86}
87
88static inline void breakout_cerrd(unsigned int val)
89{
90 switch (val & CP0_CERRD_CAUSES) {
91 case CP0_CERRD_LOAD:
92 printk(" load,");
93 break;
94 case CP0_CERRD_STORE:
95 printk(" store,");
96 break;
97 case CP0_CERRD_FILLWB:
98 printk(" fill/wb,");
99 break;
100 case CP0_CERRD_COHERENCY:
101 printk(" coherency,");
102 break;
103 case CP0_CERRD_DUPTAG:
104 printk(" duptags,");
105 break;
106 default:
107 printk(" NO CAUSE,");
108 break;
109 }
110 if (!(val & CP0_CERRD_TYPES))
111 printk(" NO TYPE");
112 else {
113 if (val & CP0_CERRD_MULTIPLE)
114 printk(" multi-err");
115 if (val & CP0_CERRD_TAG_STATE)
116 printk(" tag-state");
117 if (val & CP0_CERRD_TAG_ADDRESS)
118 printk(" tag-address");
119 if (val & CP0_CERRD_DATA_SBE)
120 printk(" data-SBE");
121 if (val & CP0_CERRD_DATA_DBE)
122 printk(" data-DBE");
123 if (val & CP0_CERRD_EXTERNAL)
124 printk(" external");
125 }
126 printk("\n");
127}
128
129#ifndef CONFIG_SIBYTE_BUS_WATCHER
130
131static void check_bus_watcher(void)
132{
133 uint32_t status, l2_err, memio_err;
134#ifdef DUMP_L2_ECC_TAG_ON_ERROR
135 uint64_t l2_tag;
136#endif
137
138 /* Destructive read, clears register and interrupt */
139 status = csr_in32(IOADDR(A_SCD_BUS_ERR_STATUS));
140 /* Bit 31 is always on, but there's no #define for that */
141 if (status & ~(1UL << 31)) {
142 l2_err = csr_in32(IOADDR(A_BUS_L2_ERRORS));
143#ifdef DUMP_L2_ECC_TAG_ON_ERROR
144 l2_tag = in64(IOADDR(A_L2_ECC_TAG));
145#endif
146 memio_err = csr_in32(IOADDR(A_BUS_MEM_IO_ERRORS));
147 printk("Bus watcher error counters: %08x %08x\n", l2_err, memio_err);
148 printk("\nLast recorded signature:\n");
149 printk("Request %02x from %d, answered by %d with Dcode %d\n",
150 (unsigned int)(G_SCD_BERR_TID(status) & 0x3f),
151 (int)(G_SCD_BERR_TID(status) >> 6),
152 (int)G_SCD_BERR_RID(status),
153 (int)G_SCD_BERR_DCODE(status));
154#ifdef DUMP_L2_ECC_TAG_ON_ERROR
155 printk("Last L2 tag w/ bad ECC: %016llx\n", l2_tag);
156#endif
157 } else {
158 printk("Bus watcher indicates no error\n");
159 }
160}
161#else
162extern void check_bus_watcher(void);
163#endif
164
165asmlinkage void sb1_cache_error(void)
166{
167 uint32_t errctl, cerr_i, cerr_d, dpalo, dpahi, eepc, res;
168 unsigned long long cerr_dpa;
169
170#ifdef CONFIG_SIBYTE_BW_TRACE
171 /* Freeze the trace buffer now */
172 csr_out32(M_SCD_TRACE_CFG_FREEZE, IOADDR(A_SCD_TRACE_CFG));
173 printk("Trace buffer frozen\n");
174#endif
175
176 printk("Cache error exception on CPU %x:\n",
177 (read_c0_prid() >> 25) & 0x7);
178
179 __asm__ __volatile__ (
180 " .set push\n\t"
181 " .set mips64\n\t"
182 " .set noat\n\t"
183 " mfc0 %0, $26\n\t"
184 " mfc0 %1, $27\n\t"
185 " mfc0 %2, $27, 1\n\t"
186 " dmfc0 $1, $27, 3\n\t"
187 " dsrl32 %3, $1, 0 \n\t"
188 " sll %4, $1, 0 \n\t"
189 " mfc0 %5, $30\n\t"
190 " .set pop"
191 : "=r" (errctl), "=r" (cerr_i), "=r" (cerr_d),
192 "=r" (dpahi), "=r" (dpalo), "=r" (eepc));
193
194 cerr_dpa = (((uint64_t)dpahi) << 32) | dpalo;
195 printk(" c0_errorepc == %08x\n", eepc);
196 printk(" c0_errctl == %08x", errctl);
197 breakout_errctl(errctl);
198 if (errctl & CP0_ERRCTL_ICACHE) {
199 printk(" c0_cerr_i == %08x", cerr_i);
200 breakout_cerri(cerr_i);
201 if (CP0_CERRI_IDX_VALID(cerr_i)) {
202 /* Check index of EPC, allowing for delay slot */
203 if (((eepc & SB1_CACHE_INDEX_MASK) != (cerr_i & SB1_CACHE_INDEX_MASK)) &&
204 ((eepc & SB1_CACHE_INDEX_MASK) != ((cerr_i & SB1_CACHE_INDEX_MASK) - 4)))
205 printk(" cerr_i idx doesn't match eepc\n");
206 else {
207 res = extract_ic(cerr_i & SB1_CACHE_INDEX_MASK,
208 (cerr_i & CP0_CERRI_DATA) != 0);
209 if (!(res & cerr_i))
210 printk("...didn't see indicated icache problem\n");
211 }
212 }
213 }
214 if (errctl & CP0_ERRCTL_DCACHE) {
215 printk(" c0_cerr_d == %08x", cerr_d);
216 breakout_cerrd(cerr_d);
217 if (CP0_CERRD_DPA_VALID(cerr_d)) {
218 printk(" c0_cerr_dpa == %010llx\n", cerr_dpa);
219 if (!CP0_CERRD_IDX_VALID(cerr_d)) {
220 res = extract_dc(cerr_dpa & SB1_CACHE_INDEX_MASK,
221 (cerr_d & CP0_CERRD_DATA) != 0);
222 if (!(res & cerr_d))
223 printk("...didn't see indicated dcache problem\n");
224 } else {
225 if ((cerr_dpa & SB1_CACHE_INDEX_MASK) != (cerr_d & SB1_CACHE_INDEX_MASK))
226 printk(" cerr_d idx doesn't match cerr_dpa\n");
227 else {
228 res = extract_dc(cerr_d & SB1_CACHE_INDEX_MASK,
229 (cerr_d & CP0_CERRD_DATA) != 0);
230 if (!(res & cerr_d))
231 printk("...didn't see indicated problem\n");
232 }
233 }
234 }
235 }
236
237 check_bus_watcher();
238
239 /*
240 * Calling panic() when a fatal cache error occurs scrambles the
241 * state of the system (and the cache), making it difficult to
242 * investigate after the fact. However, if you just stall the CPU,
243 * the other CPU may keep on running, which is typically very
244 * undesirable.
245 */
246#ifdef CONFIG_SB1_CERR_STALL
247 while (1)
248 ;
249#else
250 panic("unhandled cache error");
251#endif
252}
253
254
255/* Parity lookup table. */
256static const uint8_t parity[256] = {
257 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
258 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
259 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
260 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
261 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
262 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
263 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
264 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
265 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
266 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
267 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
268 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
269 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
270 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
271 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
272 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0
273};
274
275/* Masks to select bits for Hamming parity, mask_72_64[i] for bit[i] */
276static const uint64_t mask_72_64[8] = {
277 0x0738C808099264FFULL,
278 0x38C808099264FF07ULL,
279 0xC808099264FF0738ULL,
280 0x08099264FF0738C8ULL,
281 0x099264FF0738C808ULL,
282 0x9264FF0738C80809ULL,
283 0x64FF0738C8080992ULL,
284 0xFF0738C808099264ULL
285};
286
287/* Calculate the parity on a range of bits */
288static char range_parity(uint64_t dword, int max, int min)
289{
290 char parity = 0;
291 int i;
292 dword >>= min;
293 for (i=max-min; i>=0; i--) {
294 if (dword & 0x1)
295 parity = !parity;
296 dword >>= 1;
297 }
298 return parity;
299}
300
301/* Calculate the 4-bit even byte-parity for an instruction */
302static unsigned char inst_parity(uint32_t word)
303{
304 int i, j;
305 char parity = 0;
306 for (j=0; j<4; j++) {
307 char byte_parity = 0;
308 for (i=0; i<8; i++) {
309 if (word & 0x80000000)
310 byte_parity = !byte_parity;
311 word <<= 1;
312 }
313 parity <<= 1;
314 parity |= byte_parity;
315 }
316 return parity;
317}
318
319static uint32_t extract_ic(unsigned short addr, int data)
320{
321 unsigned short way;
322 int valid;
323 uint32_t taghi, taglolo, taglohi;
324 unsigned long long taglo, va;
325 uint64_t tlo_tmp;
326 uint8_t lru;
327 int res = 0;
328
329 printk("Icache index 0x%04x ", addr);
330 for (way = 0; way < 4; way++) {
331 /* Index-load-tag-I */
332 __asm__ __volatile__ (
333 " .set push \n\t"
334 " .set noreorder \n\t"
335 " .set mips64 \n\t"
336 " .set noat \n\t"
337 " cache 4, 0(%3) \n\t"
338 " mfc0 %0, $29 \n\t"
339 " dmfc0 $1, $28 \n\t"
340 " dsrl32 %1, $1, 0 \n\t"
341 " sll %2, $1, 0 \n\t"
342 " .set pop"
343 : "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
344 : "r" ((way << 13) | addr));
345
346 taglo = ((unsigned long long)taglohi << 32) | taglolo;
347 if (way == 0) {
348 lru = (taghi >> 14) & 0xff;
349 printk("[Bank %d Set 0x%02x] LRU > %d %d %d %d > MRU\n",
350 ((addr >> 5) & 0x3), /* bank */
351 ((addr >> 7) & 0x3f), /* index */
352 (lru & 0x3),
353 ((lru >> 2) & 0x3),
354 ((lru >> 4) & 0x3),
355 ((lru >> 6) & 0x3));
356 }
357 va = (taglo & 0xC0000FFFFFFFE000ULL) | addr;
358 if ((taglo & (1 << 31)) && (((taglo >> 62) & 0x3) == 3))
359 va |= 0x3FFFF00000000000ULL;
360 valid = ((taghi >> 29) & 1);
361 if (valid) {
362 tlo_tmp = taglo & 0xfff3ff;
363 if (((taglo >> 10) & 1) ^ range_parity(tlo_tmp, 23, 0)) {
364 printk(" ** bad parity in VTag0/G/ASID\n");
365 res |= CP0_CERRI_TAG_PARITY;
366 }
367 if (((taglo >> 11) & 1) ^ range_parity(taglo, 63, 24)) {
368 printk(" ** bad parity in R/VTag1\n");
369 res |= CP0_CERRI_TAG_PARITY;
370 }
371 }
372 if (valid ^ ((taghi >> 27) & 1)) {
373 printk(" ** bad parity for valid bit\n");
374 res |= CP0_CERRI_TAG_PARITY;
375 }
376 printk(" %d [VA %016llx] [Vld? %d] raw tags: %08X-%016llX\n",
377 way, va, valid, taghi, taglo);
378
379 if (data) {
380 uint32_t datahi, insta, instb;
381 uint8_t predecode;
382 int offset;
383
384 /* (hit all banks and ways) */
385 for (offset = 0; offset < 4; offset++) {
386 /* Index-load-data-I */
387 __asm__ __volatile__ (
388 " .set push\n\t"
389 " .set noreorder\n\t"
390 " .set mips64\n\t"
391 " .set noat\n\t"
392 " cache 6, 0(%3) \n\t"
393 " mfc0 %0, $29, 1\n\t"
394 " dmfc0 $1, $28, 1\n\t"
395 " dsrl32 %1, $1, 0 \n\t"
396 " sll %2, $1, 0 \n\t"
397 " .set pop \n"
398 : "=r" (datahi), "=r" (insta), "=r" (instb)
399 : "r" ((way << 13) | addr | (offset << 3)));
400 predecode = (datahi >> 8) & 0xff;
401 if (((datahi >> 16) & 1) != (uint32_t)range_parity(predecode, 7, 0)) {
402 printk(" ** bad parity in predecode\n");
403 res |= CP0_CERRI_DATA_PARITY;
404 }
405 /* XXXKW should/could check predecode bits themselves */
406 if (((datahi >> 4) & 0xf) ^ inst_parity(insta)) {
407 printk(" ** bad parity in instruction a\n");
408 res |= CP0_CERRI_DATA_PARITY;
409 }
410 if ((datahi & 0xf) ^ inst_parity(instb)) {
411 printk(" ** bad parity in instruction b\n");
412 res |= CP0_CERRI_DATA_PARITY;
413 }
414 printk(" %05X-%08X%08X", datahi, insta, instb);
415 }
416 printk("\n");
417 }
418 }
419 return res;
420}
421
422/* Compute the ECC for a data doubleword */
423static uint8_t dc_ecc(uint64_t dword)
424{
425 uint64_t t;
426 uint32_t w;
427 uint8_t p;
428 int i;
429
430 p = 0;
431 for (i = 7; i >= 0; i--)
432 {
433 p <<= 1;
434 t = dword & mask_72_64[i];
435 w = (uint32_t)(t >> 32);
436 p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
437 ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
438 w = (uint32_t)(t & 0xFFFFFFFF);
439 p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
440 ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
441 }
442 return p;
443}
444
445struct dc_state {
446 unsigned char val;
447 char *name;
448};
449
450static struct dc_state dc_states[] = {
451 { 0x00, "INVALID" },
452 { 0x0f, "COH-SHD" },
453 { 0x13, "NCO-E-C" },
454 { 0x19, "NCO-E-D" },
455 { 0x16, "COH-E-C" },
456 { 0x1c, "COH-E-D" },
457 { 0xff, "*ERROR*" }
458};
459
460#define DC_TAG_VALID(state) \
461 (((state) == 0x0) || ((state) == 0xf) || ((state) == 0x13) || \
462 ((state) == 0x19) || ((state) == 0x16) || ((state) == 0x1c))
463
464static char *dc_state_str(unsigned char state)
465{
466 struct dc_state *dsc = dc_states;
467 while (dsc->val != 0xff) {
468 if (dsc->val == state)
469 break;
470 dsc++;
471 }
472 return dsc->name;
473}
474
475static uint32_t extract_dc(unsigned short addr, int data)
476{
477 int valid, way;
478 unsigned char state;
479 uint32_t taghi, taglolo, taglohi;
480 unsigned long long taglo, pa;
481 uint8_t ecc, lru;
482 int res = 0;
483
484 printk("Dcache index 0x%04x ", addr);
485 for (way = 0; way < 4; way++) {
486 __asm__ __volatile__ (
487 " .set push\n\t"
488 " .set noreorder\n\t"
489 " .set mips64\n\t"
490 " .set noat\n\t"
491 " cache 5, 0(%3)\n\t" /* Index-load-tag-D */
492 " mfc0 %0, $29, 2\n\t"
493 " dmfc0 $1, $28, 2\n\t"
494 " dsrl32 %1, $1, 0\n\t"
495 " sll %2, $1, 0\n\t"
496 " .set pop"
497 : "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
498 : "r" ((way << 13) | addr));
499
500 taglo = ((unsigned long long)taglohi << 32) | taglolo;
501 pa = (taglo & 0xFFFFFFE000ULL) | addr;
502 if (way == 0) {
503 lru = (taghi >> 14) & 0xff;
504 printk("[Bank %d Set 0x%02x] LRU > %d %d %d %d > MRU\n",
505 ((addr >> 11) & 0x2) | ((addr >> 5) & 1), /* bank */
506 ((addr >> 6) & 0x3f), /* index */
507 (lru & 0x3),
508 ((lru >> 2) & 0x3),
509 ((lru >> 4) & 0x3),
510 ((lru >> 6) & 0x3));
511 }
512 state = (taghi >> 25) & 0x1f;
513 valid = DC_TAG_VALID(state);
514 printk(" %d [PA %010llx] [state %s (%02x)] raw tags: %08X-%016llX\n",
515 way, pa, dc_state_str(state), state, taghi, taglo);
516 if (valid) {
517 if (((taglo >> 11) & 1) ^ range_parity(taglo, 39, 26)) {
518 printk(" ** bad parity in PTag1\n");
519 res |= CP0_CERRD_TAG_ADDRESS;
520 }
521 if (((taglo >> 10) & 1) ^ range_parity(taglo, 25, 13)) {
522 printk(" ** bad parity in PTag0\n");
523 res |= CP0_CERRD_TAG_ADDRESS;
524 }
525 } else {
526 res |= CP0_CERRD_TAG_STATE;
527 }
528
529 if (data) {
530 uint32_t datalohi, datalolo, datahi;
531 unsigned long long datalo;
532 int offset;
533 char bad_ecc = 0;
534
535 for (offset = 0; offset < 4; offset++) {
536 /* Index-load-data-D */
537 __asm__ __volatile__ (
538 " .set push\n\t"
539 " .set noreorder\n\t"
540 " .set mips64\n\t"
541 " .set noat\n\t"
542 " cache 7, 0(%3)\n\t" /* Index-load-data-D */
543 " mfc0 %0, $29, 3\n\t"
544 " dmfc0 $1, $28, 3\n\t"
545 " dsrl32 %1, $1, 0 \n\t"
546 " sll %2, $1, 0 \n\t"
547 " .set pop"
548 : "=r" (datahi), "=r" (datalohi), "=r" (datalolo)
549 : "r" ((way << 13) | addr | (offset << 3)));
550 datalo = ((unsigned long long)datalohi << 32) | datalolo;
551 ecc = dc_ecc(datalo);
552 if (ecc != datahi) {
553 int bits;
554 bad_ecc |= 1 << (3-offset);
555 ecc ^= datahi;
556 bits = hweight8(ecc);
557 res |= (bits == 1) ? CP0_CERRD_DATA_SBE : CP0_CERRD_DATA_DBE;
558 }
559 printk(" %02X-%016llX", datahi, datalo);
560 }
561 printk("\n");
562 if (bad_ecc)
563 printk(" dwords w/ bad ECC: %d %d %d %d\n",
564 !!(bad_ecc & 8), !!(bad_ecc & 4),
565 !!(bad_ecc & 2), !!(bad_ecc & 1));
566 }
567 }
568 return res;
569}
diff --git a/arch/mips/mm/cex-gen.S b/arch/mips/mm/cex-gen.S
new file mode 100644
index 000000000..45dff5cd4
--- /dev/null
+++ b/arch/mips/mm/cex-gen.S
@@ -0,0 +1,42 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1995 - 1999 Ralf Baechle
7 * Copyright (C) 1999 Silicon Graphics, Inc.
8 *
9 * Cache error handler
10 */
11#include <asm/asm.h>
12#include <asm/regdef.h>
13#include <asm/mipsregs.h>
14#include <asm/stackframe.h>
15
16/*
17 * Game over. Go to the button. Press gently. Swear where allowed by
18 * legislation.
19 */
20 LEAF(except_vec2_generic)
21 .set noreorder
22 .set noat
23 .set mips0
24 /*
25 * This is a very bad place to be. Our cache error
26 * detection has triggered. If we have write-back data
27 * in the cache, we may not be able to recover. As a
28 * first-order desperate measure, turn off KSEG0 cacheing.
29 */
30 mfc0 k0,CP0_CONFIG
31 li k1,~CONF_CM_CMASK
32 and k0,k0,k1
33 ori k0,k0,CONF_CM_UNCACHED
34 mtc0 k0,CP0_CONFIG
35 /* Give it a few cycles to sink in... */
36 nop
37 nop
38 nop
39
40 j cache_parity_error
41 nop
42 END(except_vec2_generic)
diff --git a/arch/mips/mm/cex-oct.S b/arch/mips/mm/cex-oct.S
new file mode 100644
index 000000000..9029092aa
--- /dev/null
+++ b/arch/mips/mm/cex-oct.S
@@ -0,0 +1,70 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2006 Cavium Networks
7 * Cache error handler
8 */
9
10#include <asm/asm.h>
11#include <asm/regdef.h>
12#include <asm/mipsregs.h>
13#include <asm/stackframe.h>
14
15/*
16 * Handle cache error. Indicate to the second level handler whether
17 * the exception is recoverable.
18 */
19 LEAF(except_vec2_octeon)
20
21 .set push
22 .set mips64r2
23 .set noreorder
24 .set noat
25
26
27 /* due to an errata we need to read the COP0 CacheErr (Dcache)
28 * before any cache/DRAM access */
29
30 rdhwr k0, $0 /* get core_id */
31 PTR_LA k1, cache_err_dcache
32 sll k0, k0, 3
33 PTR_ADDU k1, k0, k1 /* k1 = &cache_err_dcache[core_id] */
34
35 dmfc0 k0, CP0_CACHEERR, 1
36 sd k0, (k1)
37 dmtc0 $0, CP0_CACHEERR, 1
38
39 /* check whether this is a nested exception */
40 mfc0 k1, CP0_STATUS
41 andi k1, k1, ST0_EXL
42 beqz k1, 1f
43 nop
44 j cache_parity_error_octeon_non_recoverable
45 nop
46
47 /* exception is recoverable */
481: j handle_cache_err
49 nop
50
51 .set pop
52 END(except_vec2_octeon)
53
54 /* We need to jump to handle_cache_err so that the previous handler
55 * can fit within 0x80 bytes. We also move from 0xFFFFFFFFAXXXXXXX
56 * space (uncached) to the 0xFFFFFFFF8XXXXXXX space (cached). */
57 LEAF(handle_cache_err)
58 .set push
59 .set noreorder
60 .set noat
61
62 SAVE_ALL
63 KMODE
64 jal cache_parity_error_octeon_recoverable
65 nop
66 j ret_from_exception
67 nop
68
69 .set pop
70 END(handle_cache_err)
diff --git a/arch/mips/mm/cex-sb1.S b/arch/mips/mm/cex-sb1.S
new file mode 100644
index 000000000..85c6e6a40
--- /dev/null
+++ b/arch/mips/mm/cex-sb1.S
@@ -0,0 +1,157 @@
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Copyright (C) 2001,2002,2003 Broadcom Corporation
4 */
5
6#include <asm/asm.h>
7#include <asm/regdef.h>
8#include <asm/mipsregs.h>
9#include <asm/stackframe.h>
10#include <asm/cacheops.h>
11#include <asm/sibyte/board.h>
12
13#define C0_ERRCTL $26 /* CP0: Error info */
14#define C0_CERR_I $27 /* CP0: Icache error */
15#define C0_CERR_D $27,1 /* CP0: Dcache error */
16
17 /*
18 * Based on SiByte sample software cache-err/cerr.S
19 * CVS revision 1.8. Only the 'unrecoverable' case
20 * is changed.
21 */
22
23 .set mips64
24 .set noreorder
25 .set noat
26
27 /*
28 * sb1_cerr_vec: code to be copied to the Cache Error
29 * Exception vector. The code must be pushed out to memory
30 * (either by copying to Kseg0 and Kseg1 both, or by flushing
31 * the L1 and L2) since it is fetched as 0xa0000100.
32 *
33 * NOTE: Be sure this handler is at most 28 instructions long
34 * since the final 16 bytes of the exception vector memory
35 * (0x170-0x17f) are used to preserve k0, k1, and ra.
36 */
37
38LEAF(except_vec2_sb1)
39 /*
40 * If this error is recoverable, we need to exit the handler
41 * without having dirtied any registers. To do this,
42 * save/restore k0 and k1 from low memory (Useg is direct
43 * mapped while ERL=1). Note that we can't save to a
44 * CPU-specific location without ruining a register in the
45 * process. This means we are vulnerable to data corruption
46 * whenever the handler is reentered by a second CPU.
47 */
48 sd k0,0x170($0)
49 sd k1,0x178($0)
50
51#ifdef CONFIG_SB1_CEX_ALWAYS_FATAL
52 j handle_vec2_sb1
53 nop
54#else
55 /*
56 * M_ERRCTL_RECOVERABLE is bit 31, which makes it easy to tell
57 * if we can fast-path out of here for a h/w-recovered error.
58 */
59 mfc0 k1,C0_ERRCTL
60 bgtz k1,attempt_recovery
61 sll k0,k1,1
62
63recovered_dcache:
64 /*
65 * Unlock CacheErr-D (which in turn unlocks CacheErr-DPA).
66 * Ought to log the occurrence of this recovered dcache error.
67 */
68 b recovered
69 mtc0 $0,C0_CERR_D
70
71attempt_recovery:
72 /*
73 * k0 has C0_ERRCTL << 1, which puts 'DC' at bit 31. Any
74 * Dcache errors we can recover from will take more extensive
75 * processing. For now, they are considered "unrecoverable".
76 * Note that 'DC' becoming set (outside of ERL mode) will
77 * cause 'IC' to clear; so if there's an Icache error, we'll
78 * only find out about it if we recover from this error and
79 * continue executing.
80 */
81 bltz k0,unrecoverable
82 sll k0,1
83
84 /*
85 * k0 has C0_ERRCTL << 2, which puts 'IC' at bit 31. If an
86 * Icache error isn't indicated, I'm not sure why we got here.
87 * Consider that case "unrecoverable" for now.
88 */
89 bgez k0,unrecoverable
90
91attempt_icache_recovery:
92 /*
93 * External icache errors are due to uncorrectable ECC errors
94 * in the L2 cache or Memory Controller and cannot be
95 * recovered here.
96 */
97 mfc0 k0,C0_CERR_I /* delay slot */
98 li k1,1 << 26 /* ICACHE_EXTERNAL */
99 and k1,k0
100 bnez k1,unrecoverable
101 andi k0,0x1fe0
102
103 /*
104 * Since the error is internal, the 'IDX' field from
105 * CacheErr-I is valid and we can just invalidate all blocks
106 * in that set.
107 */
108 cache Index_Invalidate_I,(0<<13)(k0)
109 cache Index_Invalidate_I,(1<<13)(k0)
110 cache Index_Invalidate_I,(2<<13)(k0)
111 cache Index_Invalidate_I,(3<<13)(k0)
112
113 /* Ought to log this recovered icache error */
114
115recovered:
116 /* Restore the saved registers */
117 ld k0,0x170($0)
118 ld k1,0x178($0)
119 eret
120
121unrecoverable:
122 /* Unrecoverable Icache or Dcache error; log it and/or fail */
123 j handle_vec2_sb1
124 nop
125#endif
126
127END(except_vec2_sb1)
128
129 LEAF(handle_vec2_sb1)
130 mfc0 k0,CP0_CONFIG
131 li k1,~CONF_CM_CMASK
132 and k0,k0,k1
133 ori k0,k0,CONF_CM_UNCACHED
134 mtc0 k0,CP0_CONFIG
135
136 SSNOP
137 SSNOP
138 SSNOP
139 SSNOP
140 bnezl $0, 1f
1411:
142 mfc0 k0, CP0_STATUS
143 sll k0, k0, 3 # check CU0 (kernel?)
144 bltz k0, 2f
145 nop
146
147 /* Get a valid Kseg0 stack pointer. Any task's stack pointer
148 * will do, although if we ever want to resume execution we
149 * better not have corrupted any state. */
150 get_saved_sp
151 move sp, k1
152
1532:
154 j sb1_cache_error
155 nop
156
157 END(handle_vec2_sb1)
diff --git a/arch/mips/mm/context.c b/arch/mips/mm/context.c
new file mode 100644
index 000000000..b25564090
--- /dev/null
+++ b/arch/mips/mm/context.c
@@ -0,0 +1,291 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/atomic.h>
3#include <linux/mmu_context.h>
4#include <linux/percpu.h>
5#include <linux/spinlock.h>
6
7static DEFINE_RAW_SPINLOCK(cpu_mmid_lock);
8
9static atomic64_t mmid_version;
10static unsigned int num_mmids;
11static unsigned long *mmid_map;
12
13static DEFINE_PER_CPU(u64, reserved_mmids);
14static cpumask_t tlb_flush_pending;
15
16static bool asid_versions_eq(int cpu, u64 a, u64 b)
17{
18 return ((a ^ b) & asid_version_mask(cpu)) == 0;
19}
20
21void get_new_mmu_context(struct mm_struct *mm)
22{
23 unsigned int cpu;
24 u64 asid;
25
26 /*
27 * This function is specific to ASIDs, and should not be called when
28 * MMIDs are in use.
29 */
30 if (WARN_ON(IS_ENABLED(CONFIG_DEBUG_VM) && cpu_has_mmid))
31 return;
32
33 cpu = smp_processor_id();
34 asid = asid_cache(cpu);
35
36 if (!((asid += cpu_asid_inc()) & cpu_asid_mask(&cpu_data[cpu]))) {
37 if (cpu_has_vtag_icache)
38 flush_icache_all();
39 local_flush_tlb_all(); /* start new asid cycle */
40 }
41
42 set_cpu_context(cpu, mm, asid);
43 asid_cache(cpu) = asid;
44}
45EXPORT_SYMBOL_GPL(get_new_mmu_context);
46
47void check_mmu_context(struct mm_struct *mm)
48{
49 unsigned int cpu = smp_processor_id();
50
51 /*
52 * This function is specific to ASIDs, and should not be called when
53 * MMIDs are in use.
54 */
55 if (WARN_ON(IS_ENABLED(CONFIG_DEBUG_VM) && cpu_has_mmid))
56 return;
57
58 /* Check if our ASID is of an older version and thus invalid */
59 if (!asid_versions_eq(cpu, cpu_context(cpu, mm), asid_cache(cpu)))
60 get_new_mmu_context(mm);
61}
62EXPORT_SYMBOL_GPL(check_mmu_context);
63
64static void flush_context(void)
65{
66 u64 mmid;
67 int cpu;
68
69 /* Update the list of reserved MMIDs and the MMID bitmap */
70 bitmap_clear(mmid_map, 0, num_mmids);
71
72 /* Reserve an MMID for kmap/wired entries */
73 __set_bit(MMID_KERNEL_WIRED, mmid_map);
74
75 for_each_possible_cpu(cpu) {
76 mmid = xchg_relaxed(&cpu_data[cpu].asid_cache, 0);
77
78 /*
79 * If this CPU has already been through a
80 * rollover, but hasn't run another task in
81 * the meantime, we must preserve its reserved
82 * MMID, as this is the only trace we have of
83 * the process it is still running.
84 */
85 if (mmid == 0)
86 mmid = per_cpu(reserved_mmids, cpu);
87
88 __set_bit(mmid & cpu_asid_mask(&cpu_data[cpu]), mmid_map);
89 per_cpu(reserved_mmids, cpu) = mmid;
90 }
91
92 /*
93 * Queue a TLB invalidation for each CPU to perform on next
94 * context-switch
95 */
96 cpumask_setall(&tlb_flush_pending);
97}
98
99static bool check_update_reserved_mmid(u64 mmid, u64 newmmid)
100{
101 bool hit;
102 int cpu;
103
104 /*
105 * Iterate over the set of reserved MMIDs looking for a match.
106 * If we find one, then we can update our mm to use newmmid
107 * (i.e. the same MMID in the current generation) but we can't
108 * exit the loop early, since we need to ensure that all copies
109 * of the old MMID are updated to reflect the mm. Failure to do
110 * so could result in us missing the reserved MMID in a future
111 * generation.
112 */
113 hit = false;
114 for_each_possible_cpu(cpu) {
115 if (per_cpu(reserved_mmids, cpu) == mmid) {
116 hit = true;
117 per_cpu(reserved_mmids, cpu) = newmmid;
118 }
119 }
120
121 return hit;
122}
123
124static u64 get_new_mmid(struct mm_struct *mm)
125{
126 static u32 cur_idx = MMID_KERNEL_WIRED + 1;
127 u64 mmid, version, mmid_mask;
128
129 mmid = cpu_context(0, mm);
130 version = atomic64_read(&mmid_version);
131 mmid_mask = cpu_asid_mask(&boot_cpu_data);
132
133 if (!asid_versions_eq(0, mmid, 0)) {
134 u64 newmmid = version | (mmid & mmid_mask);
135
136 /*
137 * If our current MMID was active during a rollover, we
138 * can continue to use it and this was just a false alarm.
139 */
140 if (check_update_reserved_mmid(mmid, newmmid)) {
141 mmid = newmmid;
142 goto set_context;
143 }
144
145 /*
146 * We had a valid MMID in a previous life, so try to re-use
147 * it if possible.
148 */
149 if (!__test_and_set_bit(mmid & mmid_mask, mmid_map)) {
150 mmid = newmmid;
151 goto set_context;
152 }
153 }
154
155 /* Allocate a free MMID */
156 mmid = find_next_zero_bit(mmid_map, num_mmids, cur_idx);
157 if (mmid != num_mmids)
158 goto reserve_mmid;
159
160 /* We're out of MMIDs, so increment the global version */
161 version = atomic64_add_return_relaxed(asid_first_version(0),
162 &mmid_version);
163
164 /* Note currently active MMIDs & mark TLBs as requiring flushes */
165 flush_context();
166
167 /* We have more MMIDs than CPUs, so this will always succeed */
168 mmid = find_first_zero_bit(mmid_map, num_mmids);
169
170reserve_mmid:
171 __set_bit(mmid, mmid_map);
172 cur_idx = mmid;
173 mmid |= version;
174set_context:
175 set_cpu_context(0, mm, mmid);
176 return mmid;
177}
178
179void check_switch_mmu_context(struct mm_struct *mm)
180{
181 unsigned int cpu = smp_processor_id();
182 u64 ctx, old_active_mmid;
183 unsigned long flags;
184
185 if (!cpu_has_mmid) {
186 check_mmu_context(mm);
187 write_c0_entryhi(cpu_asid(cpu, mm));
188 goto setup_pgd;
189 }
190
191 /*
192 * MMID switch fast-path, to avoid acquiring cpu_mmid_lock when it's
193 * unnecessary.
194 *
195 * The memory ordering here is subtle. If our active_mmids is non-zero
196 * and the MMID matches the current version, then we update the CPU's
197 * asid_cache with a relaxed cmpxchg. Racing with a concurrent rollover
198 * means that either:
199 *
200 * - We get a zero back from the cmpxchg and end up waiting on
201 * cpu_mmid_lock in check_mmu_context(). Taking the lock synchronises
202 * with the rollover and so we are forced to see the updated
203 * generation.
204 *
205 * - We get a valid MMID back from the cmpxchg, which means the
206 * relaxed xchg in flush_context will treat us as reserved
207 * because atomic RmWs are totally ordered for a given location.
208 */
209 ctx = cpu_context(cpu, mm);
210 old_active_mmid = READ_ONCE(cpu_data[cpu].asid_cache);
211 if (!old_active_mmid ||
212 !asid_versions_eq(cpu, ctx, atomic64_read(&mmid_version)) ||
213 !cmpxchg_relaxed(&cpu_data[cpu].asid_cache, old_active_mmid, ctx)) {
214 raw_spin_lock_irqsave(&cpu_mmid_lock, flags);
215
216 ctx = cpu_context(cpu, mm);
217 if (!asid_versions_eq(cpu, ctx, atomic64_read(&mmid_version)))
218 ctx = get_new_mmid(mm);
219
220 WRITE_ONCE(cpu_data[cpu].asid_cache, ctx);
221 raw_spin_unlock_irqrestore(&cpu_mmid_lock, flags);
222 }
223
224 /*
225 * Invalidate the local TLB if needed. Note that we must only clear our
226 * bit in tlb_flush_pending after this is complete, so that the
227 * cpu_has_shared_ftlb_entries case below isn't misled.
228 */
229 if (cpumask_test_cpu(cpu, &tlb_flush_pending)) {
230 if (cpu_has_vtag_icache)
231 flush_icache_all();
232 local_flush_tlb_all();
233 cpumask_clear_cpu(cpu, &tlb_flush_pending);
234 }
235
236 write_c0_memorymapid(ctx & cpu_asid_mask(&boot_cpu_data));
237
238 /*
239 * If this CPU shares FTLB entries with its siblings and one or more of
240 * those siblings hasn't yet invalidated its TLB following a version
241 * increase then we need to invalidate any TLB entries for our MMID
242 * that we might otherwise pick up from a sibling.
243 *
244 * We ifdef on CONFIG_SMP because cpu_sibling_map isn't defined in
245 * CONFIG_SMP=n kernels.
246 */
247#ifdef CONFIG_SMP
248 if (cpu_has_shared_ftlb_entries &&
249 cpumask_intersects(&tlb_flush_pending, &cpu_sibling_map[cpu])) {
250 /* Ensure we operate on the new MMID */
251 mtc0_tlbw_hazard();
252
253 /*
254 * Invalidate all TLB entries associated with the new
255 * MMID, and wait for the invalidation to complete.
256 */
257 ginvt_mmid();
258 sync_ginv();
259 }
260#endif
261
262setup_pgd:
263 TLBMISS_HANDLER_SETUP_PGD(mm->pgd);
264}
265EXPORT_SYMBOL_GPL(check_switch_mmu_context);
266
267static int mmid_init(void)
268{
269 if (!cpu_has_mmid)
270 return 0;
271
272 /*
273 * Expect allocation after rollover to fail if we don't have at least
274 * one more MMID than CPUs.
275 */
276 num_mmids = asid_first_version(0);
277 WARN_ON(num_mmids <= num_possible_cpus());
278
279 atomic64_set(&mmid_version, asid_first_version(0));
280 mmid_map = kcalloc(BITS_TO_LONGS(num_mmids), sizeof(*mmid_map),
281 GFP_KERNEL);
282 if (!mmid_map)
283 panic("Failed to allocate bitmap for %u MMIDs\n", num_mmids);
284
285 /* Reserve an MMID for kmap/wired entries */
286 __set_bit(MMID_KERNEL_WIRED, mmid_map);
287
288 pr_info("MMID allocator initialised with %u entries\n", num_mmids);
289 return 0;
290}
291early_initcall(mmid_init);
diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c
new file mode 100644
index 000000000..38d3d9143
--- /dev/null
+++ b/arch/mips/mm/dma-noncoherent.c
@@ -0,0 +1,145 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
4 * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org>
5 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
6 */
7#include <linux/dma-direct.h>
8#include <linux/dma-map-ops.h>
9#include <linux/highmem.h>
10
11#include <asm/cache.h>
12#include <asm/cpu-type.h>
13#include <asm/dma-coherence.h>
14#include <asm/io.h>
15
16/*
17 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively
18 * fill random cachelines with stale data at any time, requiring an extra
19 * flush post-DMA.
20 *
21 * Warning on the terminology - Linux calls an uncached area coherent; MIPS
22 * terminology calls memory areas with hardware maintained coherency coherent.
23 *
24 * Note that the R14000 and R16000 should also be checked for in this condition.
25 * However this function is only called on non-I/O-coherent systems and only the
26 * R10000 and R12000 are used in such systems, the SGI IP28 Indigo² rsp.
27 * SGI IP32 aka O2.
28 */
29static inline bool cpu_needs_post_dma_flush(void)
30{
31 switch (boot_cpu_type()) {
32 case CPU_R10000:
33 case CPU_R12000:
34 case CPU_BMIPS5000:
35 case CPU_LOONGSON2EF:
36 return true;
37 default:
38 /*
39 * Presence of MAARs suggests that the CPU supports
40 * speculatively prefetching data, and therefore requires
41 * the post-DMA flush/invalidate.
42 */
43 return cpu_has_maar;
44 }
45}
46
47void arch_dma_prep_coherent(struct page *page, size_t size)
48{
49 dma_cache_wback_inv((unsigned long)page_address(page), size);
50}
51
52void *arch_dma_set_uncached(void *addr, size_t size)
53{
54 return (void *)(__pa(addr) + UNCAC_BASE);
55}
56
57static inline void dma_sync_virt_for_device(void *addr, size_t size,
58 enum dma_data_direction dir)
59{
60 switch (dir) {
61 case DMA_TO_DEVICE:
62 dma_cache_wback((unsigned long)addr, size);
63 break;
64 case DMA_FROM_DEVICE:
65 dma_cache_inv((unsigned long)addr, size);
66 break;
67 case DMA_BIDIRECTIONAL:
68 dma_cache_wback_inv((unsigned long)addr, size);
69 break;
70 default:
71 BUG();
72 }
73}
74
75static inline void dma_sync_virt_for_cpu(void *addr, size_t size,
76 enum dma_data_direction dir)
77{
78 switch (dir) {
79 case DMA_TO_DEVICE:
80 break;
81 case DMA_FROM_DEVICE:
82 case DMA_BIDIRECTIONAL:
83 dma_cache_inv((unsigned long)addr, size);
84 break;
85 default:
86 BUG();
87 }
88}
89
90/*
91 * A single sg entry may refer to multiple physically contiguous pages. But
92 * we still need to process highmem pages individually. If highmem is not
93 * configured then the bulk of this loop gets optimized out.
94 */
95static inline void dma_sync_phys(phys_addr_t paddr, size_t size,
96 enum dma_data_direction dir, bool for_device)
97{
98 struct page *page = pfn_to_page(paddr >> PAGE_SHIFT);
99 unsigned long offset = paddr & ~PAGE_MASK;
100 size_t left = size;
101
102 do {
103 size_t len = left;
104 void *addr;
105
106 if (PageHighMem(page)) {
107 if (offset + len > PAGE_SIZE)
108 len = PAGE_SIZE - offset;
109 }
110
111 addr = kmap_atomic(page);
112 if (for_device)
113 dma_sync_virt_for_device(addr + offset, len, dir);
114 else
115 dma_sync_virt_for_cpu(addr + offset, len, dir);
116 kunmap_atomic(addr);
117
118 offset = 0;
119 page++;
120 left -= len;
121 } while (left);
122}
123
124void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
125 enum dma_data_direction dir)
126{
127 dma_sync_phys(paddr, size, dir, true);
128}
129
130#ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU
131void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
132 enum dma_data_direction dir)
133{
134 if (cpu_needs_post_dma_flush())
135 dma_sync_phys(paddr, size, dir, false);
136}
137#endif
138
139#ifdef CONFIG_DMA_PERDEV_COHERENT
140void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
141 const struct iommu_ops *iommu, bool coherent)
142{
143 dev->dma_coherent = coherent;
144}
145#endif
diff --git a/arch/mips/mm/extable.c b/arch/mips/mm/extable.c
new file mode 100644
index 000000000..81bc8a34a
--- /dev/null
+++ b/arch/mips/mm/extable.c
@@ -0,0 +1,25 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1997, 99, 2001 - 2004 Ralf Baechle <ralf@linux-mips.org>
7 */
8#include <linux/extable.h>
9#include <linux/spinlock.h>
10#include <asm/branch.h>
11#include <linux/uaccess.h>
12
13int fixup_exception(struct pt_regs *regs)
14{
15 const struct exception_table_entry *fixup;
16
17 fixup = search_exception_tables(exception_epc(regs));
18 if (fixup) {
19 regs->cp0_epc = fixup->nextinsn;
20
21 return 1;
22 }
23
24 return 0;
25}
diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c
new file mode 100644
index 000000000..7c871b14e
--- /dev/null
+++ b/arch/mips/mm/fault.c
@@ -0,0 +1,332 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1995 - 2000 by Ralf Baechle
7 */
8#include <linux/context_tracking.h>
9#include <linux/signal.h>
10#include <linux/sched.h>
11#include <linux/interrupt.h>
12#include <linux/kernel.h>
13#include <linux/errno.h>
14#include <linux/string.h>
15#include <linux/types.h>
16#include <linux/ptrace.h>
17#include <linux/ratelimit.h>
18#include <linux/mman.h>
19#include <linux/mm.h>
20#include <linux/smp.h>
21#include <linux/kprobes.h>
22#include <linux/perf_event.h>
23#include <linux/uaccess.h>
24
25#include <asm/branch.h>
26#include <asm/mmu_context.h>
27#include <asm/ptrace.h>
28#include <asm/highmem.h> /* For VMALLOC_END */
29#include <linux/kdebug.h>
30
31int show_unhandled_signals = 1;
32
33/*
34 * This routine handles page faults. It determines the address,
35 * and the problem, and then passes it off to one of the appropriate
36 * routines.
37 */
38static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
39 unsigned long address)
40{
41 struct vm_area_struct * vma = NULL;
42 struct task_struct *tsk = current;
43 struct mm_struct *mm = tsk->mm;
44 const int field = sizeof(unsigned long) * 2;
45 int si_code;
46 vm_fault_t fault;
47 unsigned int flags = FAULT_FLAG_DEFAULT;
48
49 static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10);
50
51#if 0
52 printk("Cpu%d[%s:%d:%0*lx:%ld:%0*lx]\n", raw_smp_processor_id(),
53 current->comm, current->pid, field, address, write,
54 field, regs->cp0_epc);
55#endif
56
57#ifdef CONFIG_KPROBES
58 /*
59 * This is to notify the fault handler of the kprobes.
60 */
61 if (notify_die(DIE_PAGE_FAULT, "page fault", regs, -1,
62 current->thread.trap_nr, SIGSEGV) == NOTIFY_STOP)
63 return;
64#endif
65
66 si_code = SEGV_MAPERR;
67
68 /*
69 * We fault-in kernel-space virtual memory on-demand. The
70 * 'reference' page table is init_mm.pgd.
71 *
72 * NOTE! We MUST NOT take any locks for this case. We may
73 * be in an interrupt or a critical region, and should
74 * only copy the information from the master page table,
75 * nothing more.
76 */
77#ifdef CONFIG_64BIT
78# define VMALLOC_FAULT_TARGET no_context
79#else
80# define VMALLOC_FAULT_TARGET vmalloc_fault
81#endif
82
83 if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END))
84 goto VMALLOC_FAULT_TARGET;
85#ifdef MODULE_START
86 if (unlikely(address >= MODULE_START && address < MODULE_END))
87 goto VMALLOC_FAULT_TARGET;
88#endif
89
90 /*
91 * If we're in an interrupt or have no user
92 * context, we must not take the fault..
93 */
94 if (faulthandler_disabled() || !mm)
95 goto bad_area_nosemaphore;
96
97 if (user_mode(regs))
98 flags |= FAULT_FLAG_USER;
99
100 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
101retry:
102 mmap_read_lock(mm);
103 vma = find_vma(mm, address);
104 if (!vma)
105 goto bad_area;
106 if (vma->vm_start <= address)
107 goto good_area;
108 if (!(vma->vm_flags & VM_GROWSDOWN))
109 goto bad_area;
110 if (expand_stack(vma, address))
111 goto bad_area;
112/*
113 * Ok, we have a good vm_area for this memory access, so
114 * we can handle it..
115 */
116good_area:
117 si_code = SEGV_ACCERR;
118
119 if (write) {
120 if (!(vma->vm_flags & VM_WRITE))
121 goto bad_area;
122 flags |= FAULT_FLAG_WRITE;
123 } else {
124 if (cpu_has_rixi) {
125 if (address == regs->cp0_epc && !(vma->vm_flags & VM_EXEC)) {
126#if 0
127 pr_notice("Cpu%d[%s:%d:%0*lx:%ld:%0*lx] XI violation\n",
128 raw_smp_processor_id(),
129 current->comm, current->pid,
130 field, address, write,
131 field, regs->cp0_epc);
132#endif
133 goto bad_area;
134 }
135 if (!(vma->vm_flags & VM_READ) &&
136 exception_epc(regs) != address) {
137#if 0
138 pr_notice("Cpu%d[%s:%d:%0*lx:%ld:%0*lx] RI violation\n",
139 raw_smp_processor_id(),
140 current->comm, current->pid,
141 field, address, write,
142 field, regs->cp0_epc);
143#endif
144 goto bad_area;
145 }
146 } else {
147 if (unlikely(!vma_is_accessible(vma)))
148 goto bad_area;
149 }
150 }
151
152 /*
153 * If for any reason at all we couldn't handle the fault,
154 * make sure we exit gracefully rather than endlessly redo
155 * the fault.
156 */
157 fault = handle_mm_fault(vma, address, flags, regs);
158
159 if (fault_signal_pending(fault, regs))
160 return;
161
162 if (unlikely(fault & VM_FAULT_ERROR)) {
163 if (fault & VM_FAULT_OOM)
164 goto out_of_memory;
165 else if (fault & VM_FAULT_SIGSEGV)
166 goto bad_area;
167 else if (fault & VM_FAULT_SIGBUS)
168 goto do_sigbus;
169 BUG();
170 }
171 if (flags & FAULT_FLAG_ALLOW_RETRY) {
172 if (fault & VM_FAULT_RETRY) {
173 flags |= FAULT_FLAG_TRIED;
174
175 /*
176 * No need to mmap_read_unlock(mm) as we would
177 * have already released it in __lock_page_or_retry
178 * in mm/filemap.c.
179 */
180
181 goto retry;
182 }
183 }
184
185 mmap_read_unlock(mm);
186 return;
187
188/*
189 * Something tried to access memory that isn't in our memory map..
190 * Fix it, but check if it's kernel or user first..
191 */
192bad_area:
193 mmap_read_unlock(mm);
194
195bad_area_nosemaphore:
196 /* User mode accesses just cause a SIGSEGV */
197 if (user_mode(regs)) {
198 tsk->thread.cp0_badvaddr = address;
199 tsk->thread.error_code = write;
200 if (show_unhandled_signals &&
201 unhandled_signal(tsk, SIGSEGV) &&
202 __ratelimit(&ratelimit_state)) {
203 pr_info("do_page_fault(): sending SIGSEGV to %s for invalid %s %0*lx\n",
204 tsk->comm,
205 write ? "write access to" : "read access from",
206 field, address);
207 pr_info("epc = %0*lx in", field,
208 (unsigned long) regs->cp0_epc);
209 print_vma_addr(KERN_CONT " ", regs->cp0_epc);
210 pr_cont("\n");
211 pr_info("ra = %0*lx in", field,
212 (unsigned long) regs->regs[31]);
213 print_vma_addr(KERN_CONT " ", regs->regs[31]);
214 pr_cont("\n");
215 }
216 current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f;
217 force_sig_fault(SIGSEGV, si_code, (void __user *)address);
218 return;
219 }
220
221no_context:
222 /* Are we prepared to handle this kernel fault? */
223 if (fixup_exception(regs)) {
224 current->thread.cp0_baduaddr = address;
225 return;
226 }
227
228 /*
229 * Oops. The kernel tried to access some bad page. We'll have to
230 * terminate things with extreme prejudice.
231 */
232 bust_spinlocks(1);
233
234 printk(KERN_ALERT "CPU %d Unable to handle kernel paging request at "
235 "virtual address %0*lx, epc == %0*lx, ra == %0*lx\n",
236 raw_smp_processor_id(), field, address, field, regs->cp0_epc,
237 field, regs->regs[31]);
238 die("Oops", regs);
239
240out_of_memory:
241 /*
242 * We ran out of memory, call the OOM killer, and return the userspace
243 * (which will retry the fault, or kill us if we got oom-killed).
244 */
245 mmap_read_unlock(mm);
246 if (!user_mode(regs))
247 goto no_context;
248 pagefault_out_of_memory();
249 return;
250
251do_sigbus:
252 mmap_read_unlock(mm);
253
254 /* Kernel mode? Handle exceptions or die */
255 if (!user_mode(regs))
256 goto no_context;
257
258 /*
259 * Send a sigbus, regardless of whether we were in kernel
260 * or user mode.
261 */
262#if 0
263 printk("do_page_fault() #3: sending SIGBUS to %s for "
264 "invalid %s\n%0*lx (epc == %0*lx, ra == %0*lx)\n",
265 tsk->comm,
266 write ? "write access to" : "read access from",
267 field, address,
268 field, (unsigned long) regs->cp0_epc,
269 field, (unsigned long) regs->regs[31]);
270#endif
271 current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f;
272 tsk->thread.cp0_badvaddr = address;
273 force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address);
274
275 return;
276#ifndef CONFIG_64BIT
277vmalloc_fault:
278 {
279 /*
280 * Synchronize this task's top level page-table
281 * with the 'reference' page table.
282 *
283 * Do _not_ use "tsk" here. We might be inside
284 * an interrupt in the middle of a task switch..
285 */
286 int offset = pgd_index(address);
287 pgd_t *pgd, *pgd_k;
288 p4d_t *p4d, *p4d_k;
289 pud_t *pud, *pud_k;
290 pmd_t *pmd, *pmd_k;
291 pte_t *pte_k;
292
293 pgd = (pgd_t *) pgd_current[raw_smp_processor_id()] + offset;
294 pgd_k = init_mm.pgd + offset;
295
296 if (!pgd_present(*pgd_k))
297 goto no_context;
298 set_pgd(pgd, *pgd_k);
299
300 p4d = p4d_offset(pgd, address);
301 p4d_k = p4d_offset(pgd_k, address);
302 if (!p4d_present(*p4d_k))
303 goto no_context;
304
305 pud = pud_offset(p4d, address);
306 pud_k = pud_offset(p4d_k, address);
307 if (!pud_present(*pud_k))
308 goto no_context;
309
310 pmd = pmd_offset(pud, address);
311 pmd_k = pmd_offset(pud_k, address);
312 if (!pmd_present(*pmd_k))
313 goto no_context;
314 set_pmd(pmd, *pmd_k);
315
316 pte_k = pte_offset_kernel(pmd_k, address);
317 if (!pte_present(*pte_k))
318 goto no_context;
319 return;
320 }
321#endif
322}
323
324asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
325 unsigned long write, unsigned long address)
326{
327 enum ctx_state prev_state;
328
329 prev_state = exception_enter();
330 __do_page_fault(regs, write, address);
331 exception_exit(prev_state);
332}
diff --git a/arch/mips/mm/highmem.c b/arch/mips/mm/highmem.c
new file mode 100644
index 000000000..5fec7f45d
--- /dev/null
+++ b/arch/mips/mm/highmem.c
@@ -0,0 +1,94 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/compiler.h>
3#include <linux/init.h>
4#include <linux/export.h>
5#include <linux/highmem.h>
6#include <linux/sched.h>
7#include <linux/smp.h>
8#include <asm/fixmap.h>
9#include <asm/tlbflush.h>
10
11static pte_t *kmap_pte;
12
13unsigned long highstart_pfn, highend_pfn;
14
15void kmap_flush_tlb(unsigned long addr)
16{
17 flush_tlb_one(addr);
18}
19EXPORT_SYMBOL(kmap_flush_tlb);
20
21void *kmap_atomic_high_prot(struct page *page, pgprot_t prot)
22{
23 unsigned long vaddr;
24 int idx, type;
25
26 type = kmap_atomic_idx_push();
27 idx = type + KM_TYPE_NR*smp_processor_id();
28 vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
29#ifdef CONFIG_DEBUG_HIGHMEM
30 BUG_ON(!pte_none(*(kmap_pte - idx)));
31#endif
32 set_pte(kmap_pte-idx, mk_pte(page, prot));
33 local_flush_tlb_one((unsigned long)vaddr);
34
35 return (void*) vaddr;
36}
37EXPORT_SYMBOL(kmap_atomic_high_prot);
38
39void kunmap_atomic_high(void *kvaddr)
40{
41 unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
42 int type __maybe_unused;
43
44 if (vaddr < FIXADDR_START)
45 return;
46
47 type = kmap_atomic_idx();
48#ifdef CONFIG_DEBUG_HIGHMEM
49 {
50 int idx = type + KM_TYPE_NR * smp_processor_id();
51
52 BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx));
53
54 /*
55 * force other mappings to Oops if they'll try to access
56 * this pte without first remap it
57 */
58 pte_clear(&init_mm, vaddr, kmap_pte-idx);
59 local_flush_tlb_one(vaddr);
60 }
61#endif
62 kmap_atomic_idx_pop();
63}
64EXPORT_SYMBOL(kunmap_atomic_high);
65
66/*
67 * This is the same as kmap_atomic() but can map memory that doesn't
68 * have a struct page associated with it.
69 */
70void *kmap_atomic_pfn(unsigned long pfn)
71{
72 unsigned long vaddr;
73 int idx, type;
74
75 preempt_disable();
76 pagefault_disable();
77
78 type = kmap_atomic_idx_push();
79 idx = type + KM_TYPE_NR*smp_processor_id();
80 vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
81 set_pte(kmap_pte-idx, pfn_pte(pfn, PAGE_KERNEL));
82 flush_tlb_one(vaddr);
83
84 return (void*) vaddr;
85}
86
87void __init kmap_init(void)
88{
89 unsigned long kmap_vstart;
90
91 /* cache the first kmap pte */
92 kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
93 kmap_pte = virt_to_kpte(kmap_vstart);
94}
diff --git a/arch/mips/mm/hugetlbpage.c b/arch/mips/mm/hugetlbpage.c
new file mode 100644
index 000000000..77ffece9c
--- /dev/null
+++ b/arch/mips/mm/hugetlbpage.c
@@ -0,0 +1,81 @@
1/*
2 * MIPS Huge TLB Page Support for Kernel.
3 *
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
6 * for more details.
7 *
8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
9 * Copyright 2005, Embedded Alley Solutions, Inc.
10 * Matt Porter <mporter@embeddedalley.com>
11 * Copyright (C) 2008, 2009 Cavium Networks, Inc.
12 */
13
14#include <linux/fs.h>
15#include <linux/mm.h>
16#include <linux/hugetlb.h>
17#include <linux/pagemap.h>
18#include <linux/err.h>
19#include <linux/sysctl.h>
20#include <asm/mman.h>
21#include <asm/tlb.h>
22#include <asm/tlbflush.h>
23
24pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr,
25 unsigned long sz)
26{
27 pgd_t *pgd;
28 p4d_t *p4d;
29 pud_t *pud;
30 pte_t *pte = NULL;
31
32 pgd = pgd_offset(mm, addr);
33 p4d = p4d_alloc(mm, pgd, addr);
34 pud = pud_alloc(mm, p4d, addr);
35 if (pud)
36 pte = (pte_t *)pmd_alloc(mm, pud, addr);
37
38 return pte;
39}
40
41pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr,
42 unsigned long sz)
43{
44 pgd_t *pgd;
45 p4d_t *p4d;
46 pud_t *pud;
47 pmd_t *pmd = NULL;
48
49 pgd = pgd_offset(mm, addr);
50 if (pgd_present(*pgd)) {
51 p4d = p4d_offset(pgd, addr);
52 if (p4d_present(*p4d)) {
53 pud = pud_offset(p4d, addr);
54 if (pud_present(*pud))
55 pmd = pmd_offset(pud, addr);
56 }
57 }
58 return (pte_t *) pmd;
59}
60
61/*
62 * This function checks for proper alignment of input addr and len parameters.
63 */
64int is_aligned_hugepage_range(unsigned long addr, unsigned long len)
65{
66 if (len & ~HPAGE_MASK)
67 return -EINVAL;
68 if (addr & ~HPAGE_MASK)
69 return -EINVAL;
70 return 0;
71}
72
73int pmd_huge(pmd_t pmd)
74{
75 return (pmd_val(pmd) & _PAGE_HUGE) != 0;
76}
77
78int pud_huge(pud_t pud)
79{
80 return (pud_val(pud) & _PAGE_HUGE) != 0;
81}
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
new file mode 100644
index 000000000..07e84a774
--- /dev/null
+++ b/arch/mips/mm/init.c
@@ -0,0 +1,581 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1994 - 2000 Ralf Baechle
7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8 * Kevin D. Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com
9 * Copyright (C) 2000 MIPS Technologies, Inc. All rights reserved.
10 */
11#include <linux/bug.h>
12#include <linux/init.h>
13#include <linux/export.h>
14#include <linux/signal.h>
15#include <linux/sched.h>
16#include <linux/smp.h>
17#include <linux/kernel.h>
18#include <linux/errno.h>
19#include <linux/string.h>
20#include <linux/types.h>
21#include <linux/pagemap.h>
22#include <linux/ptrace.h>
23#include <linux/mman.h>
24#include <linux/mm.h>
25#include <linux/memblock.h>
26#include <linux/highmem.h>
27#include <linux/swap.h>
28#include <linux/proc_fs.h>
29#include <linux/pfn.h>
30#include <linux/hardirq.h>
31#include <linux/gfp.h>
32#include <linux/kcore.h>
33#include <linux/initrd.h>
34
35#include <asm/bootinfo.h>
36#include <asm/cachectl.h>
37#include <asm/cpu.h>
38#include <asm/dma.h>
39#include <asm/kmap_types.h>
40#include <asm/maar.h>
41#include <asm/mmu_context.h>
42#include <asm/sections.h>
43#include <asm/pgalloc.h>
44#include <asm/tlb.h>
45#include <asm/fixmap.h>
46
47/*
48 * We have up to 8 empty zeroed pages so we can map one of the right colour
49 * when needed. This is necessary only on R4000 / R4400 SC and MC versions
50 * where we have to avoid VCED / VECI exceptions for good performance at
51 * any price. Since page is never written to after the initialization we
52 * don't have to care about aliases on other CPUs.
53 */
54unsigned long empty_zero_page, zero_page_mask;
55EXPORT_SYMBOL_GPL(empty_zero_page);
56EXPORT_SYMBOL(zero_page_mask);
57
58/*
59 * Not static inline because used by IP27 special magic initialization code
60 */
61void setup_zero_pages(void)
62{
63 unsigned int order, i;
64 struct page *page;
65
66 if (cpu_has_vce)
67 order = 3;
68 else
69 order = 0;
70
71 empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
72 if (!empty_zero_page)
73 panic("Oh boy, that early out of memory?");
74
75 page = virt_to_page((void *)empty_zero_page);
76 split_page(page, order);
77 for (i = 0; i < (1 << order); i++, page++)
78 mark_page_reserved(page);
79
80 zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK;
81}
82
83static void *__kmap_pgprot(struct page *page, unsigned long addr, pgprot_t prot)
84{
85 enum fixed_addresses idx;
86 unsigned int old_mmid;
87 unsigned long vaddr, flags, entrylo;
88 unsigned long old_ctx;
89 pte_t pte;
90 int tlbidx;
91
92 BUG_ON(Page_dcache_dirty(page));
93
94 preempt_disable();
95 pagefault_disable();
96 idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
97 idx += in_interrupt() ? FIX_N_COLOURS : 0;
98 vaddr = __fix_to_virt(FIX_CMAP_END - idx);
99 pte = mk_pte(page, prot);
100#if defined(CONFIG_XPA)
101 entrylo = pte_to_entrylo(pte.pte_high);
102#elif defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32)
103 entrylo = pte.pte_high;
104#else
105 entrylo = pte_to_entrylo(pte_val(pte));
106#endif
107
108 local_irq_save(flags);
109 old_ctx = read_c0_entryhi();
110 write_c0_entryhi(vaddr & (PAGE_MASK << 1));
111 write_c0_entrylo0(entrylo);
112 write_c0_entrylo1(entrylo);
113 if (cpu_has_mmid) {
114 old_mmid = read_c0_memorymapid();
115 write_c0_memorymapid(MMID_KERNEL_WIRED);
116 }
117#ifdef CONFIG_XPA
118 if (cpu_has_xpa) {
119 entrylo = (pte.pte_low & _PFNX_MASK);
120 writex_c0_entrylo0(entrylo);
121 writex_c0_entrylo1(entrylo);
122 }
123#endif
124 tlbidx = num_wired_entries();
125 write_c0_wired(tlbidx + 1);
126 write_c0_index(tlbidx);
127 mtc0_tlbw_hazard();
128 tlb_write_indexed();
129 tlbw_use_hazard();
130 write_c0_entryhi(old_ctx);
131 if (cpu_has_mmid)
132 write_c0_memorymapid(old_mmid);
133 local_irq_restore(flags);
134
135 return (void*) vaddr;
136}
137
138void *kmap_coherent(struct page *page, unsigned long addr)
139{
140 return __kmap_pgprot(page, addr, PAGE_KERNEL);
141}
142
143void *kmap_noncoherent(struct page *page, unsigned long addr)
144{
145 return __kmap_pgprot(page, addr, PAGE_KERNEL_NC);
146}
147
148void kunmap_coherent(void)
149{
150 unsigned int wired;
151 unsigned long flags, old_ctx;
152
153 local_irq_save(flags);
154 old_ctx = read_c0_entryhi();
155 wired = num_wired_entries() - 1;
156 write_c0_wired(wired);
157 write_c0_index(wired);
158 write_c0_entryhi(UNIQUE_ENTRYHI(wired));
159 write_c0_entrylo0(0);
160 write_c0_entrylo1(0);
161 mtc0_tlbw_hazard();
162 tlb_write_indexed();
163 tlbw_use_hazard();
164 write_c0_entryhi(old_ctx);
165 local_irq_restore(flags);
166 pagefault_enable();
167 preempt_enable();
168}
169
170void copy_user_highpage(struct page *to, struct page *from,
171 unsigned long vaddr, struct vm_area_struct *vma)
172{
173 void *vfrom, *vto;
174
175 vto = kmap_atomic(to);
176 if (cpu_has_dc_aliases &&
177 page_mapcount(from) && !Page_dcache_dirty(from)) {
178 vfrom = kmap_coherent(from, vaddr);
179 copy_page(vto, vfrom);
180 kunmap_coherent();
181 } else {
182 vfrom = kmap_atomic(from);
183 copy_page(vto, vfrom);
184 kunmap_atomic(vfrom);
185 }
186 if ((!cpu_has_ic_fills_f_dc) ||
187 pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))
188 flush_data_cache_page((unsigned long)vto);
189 kunmap_atomic(vto);
190 /* Make sure this page is cleared on other CPU's too before using it */
191 smp_wmb();
192}
193
194void copy_to_user_page(struct vm_area_struct *vma,
195 struct page *page, unsigned long vaddr, void *dst, const void *src,
196 unsigned long len)
197{
198 if (cpu_has_dc_aliases &&
199 page_mapcount(page) && !Page_dcache_dirty(page)) {
200 void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
201 memcpy(vto, src, len);
202 kunmap_coherent();
203 } else {
204 memcpy(dst, src, len);
205 if (cpu_has_dc_aliases)
206 SetPageDcacheDirty(page);
207 }
208 if (vma->vm_flags & VM_EXEC)
209 flush_cache_page(vma, vaddr, page_to_pfn(page));
210}
211
212void copy_from_user_page(struct vm_area_struct *vma,
213 struct page *page, unsigned long vaddr, void *dst, const void *src,
214 unsigned long len)
215{
216 if (cpu_has_dc_aliases &&
217 page_mapcount(page) && !Page_dcache_dirty(page)) {
218 void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
219 memcpy(dst, vfrom, len);
220 kunmap_coherent();
221 } else {
222 memcpy(dst, src, len);
223 if (cpu_has_dc_aliases)
224 SetPageDcacheDirty(page);
225 }
226}
227EXPORT_SYMBOL_GPL(copy_from_user_page);
228
229void __init fixrange_init(unsigned long start, unsigned long end,
230 pgd_t *pgd_base)
231{
232#ifdef CONFIG_HIGHMEM
233 pgd_t *pgd;
234 pud_t *pud;
235 pmd_t *pmd;
236 pte_t *pte;
237 int i, j, k;
238 unsigned long vaddr;
239
240 vaddr = start;
241 i = pgd_index(vaddr);
242 j = pud_index(vaddr);
243 k = pmd_index(vaddr);
244 pgd = pgd_base + i;
245
246 for ( ; (i < PTRS_PER_PGD) && (vaddr < end); pgd++, i++) {
247 pud = (pud_t *)pgd;
248 for ( ; (j < PTRS_PER_PUD) && (vaddr < end); pud++, j++) {
249 pmd = (pmd_t *)pud;
250 for (; (k < PTRS_PER_PMD) && (vaddr < end); pmd++, k++) {
251 if (pmd_none(*pmd)) {
252 pte = (pte_t *) memblock_alloc_low(PAGE_SIZE,
253 PAGE_SIZE);
254 if (!pte)
255 panic("%s: Failed to allocate %lu bytes align=%lx\n",
256 __func__, PAGE_SIZE,
257 PAGE_SIZE);
258
259 set_pmd(pmd, __pmd((unsigned long)pte));
260 BUG_ON(pte != pte_offset_kernel(pmd, 0));
261 }
262 vaddr += PMD_SIZE;
263 }
264 k = 0;
265 }
266 j = 0;
267 }
268#endif
269}
270
271struct maar_walk_info {
272 struct maar_config cfg[16];
273 unsigned int num_cfg;
274};
275
276static int maar_res_walk(unsigned long start_pfn, unsigned long nr_pages,
277 void *data)
278{
279 struct maar_walk_info *wi = data;
280 struct maar_config *cfg = &wi->cfg[wi->num_cfg];
281 unsigned int maar_align;
282
283 /* MAAR registers hold physical addresses right shifted by 4 bits */
284 maar_align = BIT(MIPS_MAAR_ADDR_SHIFT + 4);
285
286 /* Fill in the MAAR config entry */
287 cfg->lower = ALIGN(PFN_PHYS(start_pfn), maar_align);
288 cfg->upper = ALIGN_DOWN(PFN_PHYS(start_pfn + nr_pages), maar_align) - 1;
289 cfg->attrs = MIPS_MAAR_S;
290
291 /* Ensure we don't overflow the cfg array */
292 if (!WARN_ON(wi->num_cfg >= ARRAY_SIZE(wi->cfg)))
293 wi->num_cfg++;
294
295 return 0;
296}
297
298
299unsigned __weak platform_maar_init(unsigned num_pairs)
300{
301 unsigned int num_configured;
302 struct maar_walk_info wi;
303
304 wi.num_cfg = 0;
305 walk_system_ram_range(0, max_pfn, &wi, maar_res_walk);
306
307 num_configured = maar_config(wi.cfg, wi.num_cfg, num_pairs);
308 if (num_configured < wi.num_cfg)
309 pr_warn("Not enough MAAR pairs (%u) for all memory regions (%u)\n",
310 num_pairs, wi.num_cfg);
311
312 return num_configured;
313}
314
315void maar_init(void)
316{
317 unsigned num_maars, used, i;
318 phys_addr_t lower, upper, attr;
319 static struct {
320 struct maar_config cfgs[3];
321 unsigned used;
322 } recorded = { { { 0 } }, 0 };
323
324 if (!cpu_has_maar)
325 return;
326
327 /* Detect the number of MAARs */
328 write_c0_maari(~0);
329 back_to_back_c0_hazard();
330 num_maars = read_c0_maari() + 1;
331
332 /* MAARs should be in pairs */
333 WARN_ON(num_maars % 2);
334
335 /* Set MAARs using values we recorded already */
336 if (recorded.used) {
337 used = maar_config(recorded.cfgs, recorded.used, num_maars / 2);
338 BUG_ON(used != recorded.used);
339 } else {
340 /* Configure the required MAARs */
341 used = platform_maar_init(num_maars / 2);
342 }
343
344 /* Disable any further MAARs */
345 for (i = (used * 2); i < num_maars; i++) {
346 write_c0_maari(i);
347 back_to_back_c0_hazard();
348 write_c0_maar(0);
349 back_to_back_c0_hazard();
350 }
351
352 if (recorded.used)
353 return;
354
355 pr_info("MAAR configuration:\n");
356 for (i = 0; i < num_maars; i += 2) {
357 write_c0_maari(i);
358 back_to_back_c0_hazard();
359 upper = read_c0_maar();
360#ifdef CONFIG_XPA
361 upper |= (phys_addr_t)readx_c0_maar() << MIPS_MAARX_ADDR_SHIFT;
362#endif
363
364 write_c0_maari(i + 1);
365 back_to_back_c0_hazard();
366 lower = read_c0_maar();
367#ifdef CONFIG_XPA
368 lower |= (phys_addr_t)readx_c0_maar() << MIPS_MAARX_ADDR_SHIFT;
369#endif
370
371 attr = lower & upper;
372 lower = (lower & MIPS_MAAR_ADDR) << 4;
373 upper = ((upper & MIPS_MAAR_ADDR) << 4) | 0xffff;
374
375 pr_info(" [%d]: ", i / 2);
376 if ((attr & MIPS_MAAR_V) != MIPS_MAAR_V) {
377 pr_cont("disabled\n");
378 continue;
379 }
380
381 pr_cont("%pa-%pa", &lower, &upper);
382
383 if (attr & MIPS_MAAR_S)
384 pr_cont(" speculate");
385
386 pr_cont("\n");
387
388 /* Record the setup for use on secondary CPUs */
389 if (used <= ARRAY_SIZE(recorded.cfgs)) {
390 recorded.cfgs[recorded.used].lower = lower;
391 recorded.cfgs[recorded.used].upper = upper;
392 recorded.cfgs[recorded.used].attrs = attr;
393 recorded.used++;
394 }
395 }
396}
397
398#ifndef CONFIG_NEED_MULTIPLE_NODES
399void __init paging_init(void)
400{
401 unsigned long max_zone_pfns[MAX_NR_ZONES];
402
403 pagetable_init();
404
405#ifdef CONFIG_HIGHMEM
406 kmap_init();
407#endif
408#ifdef CONFIG_ZONE_DMA
409 max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
410#endif
411#ifdef CONFIG_ZONE_DMA32
412 max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
413#endif
414 max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
415#ifdef CONFIG_HIGHMEM
416 max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;
417
418 if (cpu_has_dc_aliases && max_low_pfn != highend_pfn) {
419 printk(KERN_WARNING "This processor doesn't support highmem."
420 " %ldk highmem ignored\n",
421 (highend_pfn - max_low_pfn) << (PAGE_SHIFT - 10));
422 max_zone_pfns[ZONE_HIGHMEM] = max_low_pfn;
423 }
424#endif
425
426 free_area_init(max_zone_pfns);
427}
428
429#ifdef CONFIG_64BIT
430static struct kcore_list kcore_kseg0;
431#endif
432
433static inline void __init mem_init_free_highmem(void)
434{
435#ifdef CONFIG_HIGHMEM
436 unsigned long tmp;
437
438 if (cpu_has_dc_aliases)
439 return;
440
441 for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
442 struct page *page = pfn_to_page(tmp);
443
444 if (!memblock_is_memory(PFN_PHYS(tmp)))
445 SetPageReserved(page);
446 else
447 free_highmem_page(page);
448 }
449#endif
450}
451
452void __init mem_init(void)
453{
454 /*
455 * When _PFN_SHIFT is greater than PAGE_SHIFT we won't have enough PTE
456 * bits to hold a full 32b physical address on MIPS32 systems.
457 */
458 BUILD_BUG_ON(IS_ENABLED(CONFIG_32BIT) && (_PFN_SHIFT > PAGE_SHIFT));
459
460#ifdef CONFIG_HIGHMEM
461#ifdef CONFIG_DISCONTIGMEM
462#error "CONFIG_HIGHMEM and CONFIG_DISCONTIGMEM dont work together yet"
463#endif
464 max_mapnr = highend_pfn ? highend_pfn : max_low_pfn;
465#else
466 max_mapnr = max_low_pfn;
467#endif
468 high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
469
470 maar_init();
471 memblock_free_all();
472 setup_zero_pages(); /* Setup zeroed pages. */
473 mem_init_free_highmem();
474 mem_init_print_info(NULL);
475
476#ifdef CONFIG_64BIT
477 if ((unsigned long) &_text > (unsigned long) CKSEG0)
478 /* The -4 is a hack so that user tools don't have to handle
479 the overflow. */
480 kclist_add(&kcore_kseg0, (void *) CKSEG0,
481 0x80000000 - 4, KCORE_TEXT);
482#endif
483}
484#endif /* !CONFIG_NEED_MULTIPLE_NODES */
485
486void free_init_pages(const char *what, unsigned long begin, unsigned long end)
487{
488 unsigned long pfn;
489
490 for (pfn = PFN_UP(begin); pfn < PFN_DOWN(end); pfn++) {
491 struct page *page = pfn_to_page(pfn);
492 void *addr = phys_to_virt(PFN_PHYS(pfn));
493
494 memset(addr, POISON_FREE_INITMEM, PAGE_SIZE);
495 free_reserved_page(page);
496 }
497 printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
498}
499
500void (*free_init_pages_eva)(void *begin, void *end) = NULL;
501
502void __ref free_initmem(void)
503{
504 prom_free_prom_memory();
505 /*
506 * Let the platform define a specific function to free the
507 * init section since EVA may have used any possible mapping
508 * between virtual and physical addresses.
509 */
510 if (free_init_pages_eva)
511 free_init_pages_eva((void *)&__init_begin, (void *)&__init_end);
512 else
513 free_initmem_default(POISON_FREE_INITMEM);
514}
515
516#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
517unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
518EXPORT_SYMBOL(__per_cpu_offset);
519
520static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
521{
522 return node_distance(cpu_to_node(from), cpu_to_node(to));
523}
524
525static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size,
526 size_t align)
527{
528 return memblock_alloc_try_nid(size, align, __pa(MAX_DMA_ADDRESS),
529 MEMBLOCK_ALLOC_ACCESSIBLE,
530 cpu_to_node(cpu));
531}
532
533static void __init pcpu_fc_free(void *ptr, size_t size)
534{
535 memblock_free_early(__pa(ptr), size);
536}
537
538void __init setup_per_cpu_areas(void)
539{
540 unsigned long delta;
541 unsigned int cpu;
542 int rc;
543
544 /*
545 * Always reserve area for module percpu variables. That's
546 * what the legacy allocator did.
547 */
548 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
549 PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,
550 pcpu_cpu_distance,
551 pcpu_fc_alloc, pcpu_fc_free);
552 if (rc < 0)
553 panic("Failed to initialize percpu areas.");
554
555 delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
556 for_each_possible_cpu(cpu)
557 __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
558}
559#endif
560
561#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
562unsigned long pgd_current[NR_CPUS];
563#endif
564
565/*
566 * Align swapper_pg_dir in to 64K, allows its address to be loaded
567 * with a single LUI instruction in the TLB handlers. If we used
568 * __aligned(64K), its size would get rounded up to the alignment
569 * size, and waste space. So we place it in its own section and align
570 * it in the linker script.
571 */
572pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir");
573#ifndef __PAGETABLE_PUD_FOLDED
574pud_t invalid_pud_table[PTRS_PER_PUD] __page_aligned_bss;
575#endif
576#ifndef __PAGETABLE_PMD_FOLDED
577pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
578EXPORT_SYMBOL_GPL(invalid_pmd_table);
579#endif
580pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
581EXPORT_SYMBOL(invalid_pte_table);
diff --git a/arch/mips/mm/ioremap.c b/arch/mips/mm/ioremap.c
new file mode 100644
index 000000000..b6dad2fd5
--- /dev/null
+++ b/arch/mips/mm/ioremap.c
@@ -0,0 +1,119 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * (C) Copyright 1995 1996 Linus Torvalds
7 * (C) Copyright 2001, 2002 Ralf Baechle
8 */
9#include <linux/export.h>
10#include <asm/addrspace.h>
11#include <asm/byteorder.h>
12#include <linux/ioport.h>
13#include <linux/sched.h>
14#include <linux/slab.h>
15#include <linux/vmalloc.h>
16#include <linux/mm_types.h>
17#include <linux/io.h>
18#include <asm/cacheflush.h>
19#include <asm/tlbflush.h>
20#include <ioremap.h>
21
22#define IS_LOW512(addr) (!((phys_addr_t)(addr) & (phys_addr_t) ~0x1fffffffULL))
23#define IS_KSEG1(addr) (((unsigned long)(addr) & ~0x1fffffffUL) == CKSEG1)
24
25static int __ioremap_check_ram(unsigned long start_pfn, unsigned long nr_pages,
26 void *arg)
27{
28 unsigned long i;
29
30 for (i = 0; i < nr_pages; i++) {
31 if (pfn_valid(start_pfn + i) &&
32 !PageReserved(pfn_to_page(start_pfn + i)))
33 return 1;
34 }
35
36 return 0;
37}
38
39/*
40 * ioremap_prot - map bus memory into CPU space
41 * @phys_addr: bus address of the memory
42 * @size: size of the resource to map
43 *
44 * ioremap_prot gives the caller control over cache coherency attributes (CCA)
45 */
46void __iomem *ioremap_prot(phys_addr_t phys_addr, unsigned long size,
47 unsigned long prot_val)
48{
49 unsigned long flags = prot_val & _CACHE_MASK;
50 unsigned long offset, pfn, last_pfn;
51 struct vm_struct *area;
52 phys_addr_t last_addr;
53 unsigned long vaddr;
54 void __iomem *cpu_addr;
55
56 cpu_addr = plat_ioremap(phys_addr, size, flags);
57 if (cpu_addr)
58 return cpu_addr;
59
60 phys_addr = fixup_bigphys_addr(phys_addr, size);
61
62 /* Don't allow wraparound or zero size */
63 last_addr = phys_addr + size - 1;
64 if (!size || last_addr < phys_addr)
65 return NULL;
66
67 /*
68 * Map uncached objects in the low 512mb of address space using KSEG1,
69 * otherwise map using page tables.
70 */
71 if (IS_LOW512(phys_addr) && IS_LOW512(last_addr) &&
72 flags == _CACHE_UNCACHED)
73 return (void __iomem *) CKSEG1ADDR(phys_addr);
74
75 /*
76 * Don't allow anybody to remap RAM that may be allocated by the page
77 * allocator, since that could lead to races & data clobbering.
78 */
79 pfn = PFN_DOWN(phys_addr);
80 last_pfn = PFN_DOWN(last_addr);
81 if (walk_system_ram_range(pfn, last_pfn - pfn + 1, NULL,
82 __ioremap_check_ram) == 1) {
83 WARN_ONCE(1, "ioremap on RAM at %pa - %pa\n",
84 &phys_addr, &last_addr);
85 return NULL;
86 }
87
88 /*
89 * Mappings have to be page-aligned
90 */
91 offset = phys_addr & ~PAGE_MASK;
92 phys_addr &= PAGE_MASK;
93 size = PAGE_ALIGN(last_addr + 1) - phys_addr;
94
95 /*
96 * Ok, go for it..
97 */
98 area = get_vm_area(size, VM_IOREMAP);
99 if (!area)
100 return NULL;
101 vaddr = (unsigned long)area->addr;
102
103 flags |= _PAGE_GLOBAL | _PAGE_PRESENT | __READABLE | __WRITEABLE;
104 if (ioremap_page_range(vaddr, vaddr + size, phys_addr,
105 __pgprot(flags))) {
106 free_vm_area(area);
107 return NULL;
108 }
109
110 return (void __iomem *)(vaddr + offset);
111}
112EXPORT_SYMBOL(ioremap_prot);
113
114void iounmap(const volatile void __iomem *addr)
115{
116 if (!plat_iounmap(addr) && !IS_KSEG1(addr))
117 vunmap((void *)((unsigned long)addr & PAGE_MASK));
118}
119EXPORT_SYMBOL(iounmap);
diff --git a/arch/mips/mm/ioremap64.c b/arch/mips/mm/ioremap64.c
new file mode 100644
index 000000000..15e7820d6
--- /dev/null
+++ b/arch/mips/mm/ioremap64.c
@@ -0,0 +1,23 @@
1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/io.h>
3#include <ioremap.h>
4
5void __iomem *ioremap_prot(phys_addr_t offset, unsigned long size,
6 unsigned long prot_val)
7{
8 unsigned long flags = prot_val & _CACHE_MASK;
9 u64 base = (flags == _CACHE_UNCACHED ? IO_BASE : UNCAC_BASE);
10 void __iomem *addr;
11
12 addr = plat_ioremap(offset, size, flags);
13 if (!addr)
14 addr = (void __iomem *)(unsigned long)(base + offset);
15 return addr;
16}
17EXPORT_SYMBOL(ioremap_prot);
18
19void iounmap(const volatile void __iomem *addr)
20{
21 plat_iounmap(addr);
22}
23EXPORT_SYMBOL(iounmap);
diff --git a/arch/mips/mm/mmap.c b/arch/mips/mm/mmap.c
new file mode 100644
index 000000000..00fe90c6d
--- /dev/null
+++ b/arch/mips/mm/mmap.c
@@ -0,0 +1,129 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2011 Wind River Systems,
7 * written by Ralf Baechle <ralf@linux-mips.org>
8 */
9#include <linux/compiler.h>
10#include <linux/elf-randomize.h>
11#include <linux/errno.h>
12#include <linux/mm.h>
13#include <linux/mman.h>
14#include <linux/export.h>
15#include <linux/personality.h>
16#include <linux/random.h>
17#include <linux/sched/signal.h>
18#include <linux/sched/mm.h>
19
20unsigned long shm_align_mask = PAGE_SIZE - 1; /* Sane caches */
21EXPORT_SYMBOL(shm_align_mask);
22
23#define COLOUR_ALIGN(addr, pgoff) \
24 ((((addr) + shm_align_mask) & ~shm_align_mask) + \
25 (((pgoff) << PAGE_SHIFT) & shm_align_mask))
26
27enum mmap_allocation_direction {UP, DOWN};
28
29static unsigned long arch_get_unmapped_area_common(struct file *filp,
30 unsigned long addr0, unsigned long len, unsigned long pgoff,
31 unsigned long flags, enum mmap_allocation_direction dir)
32{
33 struct mm_struct *mm = current->mm;
34 struct vm_area_struct *vma;
35 unsigned long addr = addr0;
36 int do_color_align;
37 struct vm_unmapped_area_info info;
38
39 if (unlikely(len > TASK_SIZE))
40 return -ENOMEM;
41
42 if (flags & MAP_FIXED) {
43 /* Even MAP_FIXED mappings must reside within TASK_SIZE */
44 if (TASK_SIZE - len < addr)
45 return -EINVAL;
46
47 /*
48 * We do not accept a shared mapping if it would violate
49 * cache aliasing constraints.
50 */
51 if ((flags & MAP_SHARED) &&
52 ((addr - (pgoff << PAGE_SHIFT)) & shm_align_mask))
53 return -EINVAL;
54 return addr;
55 }
56
57 do_color_align = 0;
58 if (filp || (flags & MAP_SHARED))
59 do_color_align = 1;
60
61 /* requesting a specific address */
62 if (addr) {
63 if (do_color_align)
64 addr = COLOUR_ALIGN(addr, pgoff);
65 else
66 addr = PAGE_ALIGN(addr);
67
68 vma = find_vma(mm, addr);
69 if (TASK_SIZE - len >= addr &&
70 (!vma || addr + len <= vm_start_gap(vma)))
71 return addr;
72 }
73
74 info.length = len;
75 info.align_mask = do_color_align ? (PAGE_MASK & shm_align_mask) : 0;
76 info.align_offset = pgoff << PAGE_SHIFT;
77
78 if (dir == DOWN) {
79 info.flags = VM_UNMAPPED_AREA_TOPDOWN;
80 info.low_limit = PAGE_SIZE;
81 info.high_limit = mm->mmap_base;
82 addr = vm_unmapped_area(&info);
83
84 if (!(addr & ~PAGE_MASK))
85 return addr;
86
87 /*
88 * A failed mmap() very likely causes application failure,
89 * so fall back to the bottom-up function here. This scenario
90 * can happen with large stack limits and large mmap()
91 * allocations.
92 */
93 }
94
95 info.flags = 0;
96 info.low_limit = mm->mmap_base;
97 info.high_limit = TASK_SIZE;
98 return vm_unmapped_area(&info);
99}
100
101unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr0,
102 unsigned long len, unsigned long pgoff, unsigned long flags)
103{
104 return arch_get_unmapped_area_common(filp,
105 addr0, len, pgoff, flags, UP);
106}
107
108/*
109 * There is no need to export this but sched.h declares the function as
110 * extern so making it static here results in an error.
111 */
112unsigned long arch_get_unmapped_area_topdown(struct file *filp,
113 unsigned long addr0, unsigned long len, unsigned long pgoff,
114 unsigned long flags)
115{
116 return arch_get_unmapped_area_common(filp,
117 addr0, len, pgoff, flags, DOWN);
118}
119
120bool __virt_addr_valid(const volatile void *kaddr)
121{
122 unsigned long vaddr = (unsigned long)kaddr;
123
124 if ((vaddr < PAGE_OFFSET) || (vaddr >= MAP_BASE))
125 return false;
126
127 return pfn_valid(PFN_DOWN(virt_to_phys(kaddr)));
128}
129EXPORT_SYMBOL_GPL(__virt_addr_valid);
diff --git a/arch/mips/mm/page-funcs.S b/arch/mips/mm/page-funcs.S
new file mode 100644
index 000000000..43181ac0a
--- /dev/null
+++ b/arch/mips/mm/page-funcs.S
@@ -0,0 +1,53 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Micro-assembler generated clear_page/copy_page functions.
7 *
8 * Copyright (C) 2012 MIPS Technologies, Inc.
9 * Copyright (C) 2012 Ralf Baechle <ralf@linux-mips.org>
10 */
11#include <asm/asm.h>
12#include <asm/export.h>
13#include <asm/regdef.h>
14
15#ifdef CONFIG_SIBYTE_DMA_PAGEOPS
16#define cpu_clear_page_function_name clear_page_cpu
17#define cpu_copy_page_function_name copy_page_cpu
18#else
19#define cpu_clear_page_function_name clear_page
20#define cpu_copy_page_function_name copy_page
21#endif
22
23/*
24 * Maximum sizes:
25 *
26 * R4000 128 bytes S-cache: 0x058 bytes
27 * R4600 v1.7: 0x05c bytes
28 * R4600 v2.0: 0x060 bytes
29 * With prefetching, 16 word strides 0x120 bytes
30 */
31EXPORT(__clear_page_start)
32LEAF(cpu_clear_page_function_name)
33EXPORT_SYMBOL(cpu_clear_page_function_name)
341: j 1b /* Dummy, will be replaced. */
35 .space 288
36END(cpu_clear_page_function_name)
37EXPORT(__clear_page_end)
38
39/*
40 * Maximum sizes:
41 *
42 * R4000 128 bytes S-cache: 0x11c bytes
43 * R4600 v1.7: 0x080 bytes
44 * R4600 v2.0: 0x07c bytes
45 * With prefetching, 16 word strides 0x540 bytes
46 */
47EXPORT(__copy_page_start)
48LEAF(cpu_copy_page_function_name)
49EXPORT_SYMBOL(cpu_copy_page_function_name)
501: j 1b /* Dummy, will be replaced. */
51 .space 1344
52END(cpu_copy_page_function_name)
53EXPORT(__copy_page_end)
diff --git a/arch/mips/mm/page.c b/arch/mips/mm/page.c
new file mode 100644
index 000000000..504bc4047
--- /dev/null
+++ b/arch/mips/mm/page.c
@@ -0,0 +1,681 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (C) 2007 Maciej W. Rozycki
8 * Copyright (C) 2008 Thiemo Seufer
9 * Copyright (C) 2012 MIPS Technologies, Inc.
10 */
11#include <linux/kernel.h>
12#include <linux/sched.h>
13#include <linux/smp.h>
14#include <linux/mm.h>
15#include <linux/proc_fs.h>
16
17#include <asm/bugs.h>
18#include <asm/cacheops.h>
19#include <asm/cpu-type.h>
20#include <asm/inst.h>
21#include <asm/io.h>
22#include <asm/page.h>
23#include <asm/prefetch.h>
24#include <asm/bootinfo.h>
25#include <asm/mipsregs.h>
26#include <asm/mmu_context.h>
27#include <asm/cpu.h>
28#include <asm/war.h>
29
30#ifdef CONFIG_SIBYTE_DMA_PAGEOPS
31#include <asm/sibyte/sb1250.h>
32#include <asm/sibyte/sb1250_regs.h>
33#include <asm/sibyte/sb1250_dma.h>
34#endif
35
36#include <asm/uasm.h>
37
38/* Registers used in the assembled routines. */
39#define ZERO 0
40#define AT 2
41#define A0 4
42#define A1 5
43#define A2 6
44#define T0 8
45#define T1 9
46#define T2 10
47#define T3 11
48#define T9 25
49#define RA 31
50
51/* Handle labels (which must be positive integers). */
52enum label_id {
53 label_clear_nopref = 1,
54 label_clear_pref,
55 label_copy_nopref,
56 label_copy_pref_both,
57 label_copy_pref_store,
58};
59
60UASM_L_LA(_clear_nopref)
61UASM_L_LA(_clear_pref)
62UASM_L_LA(_copy_nopref)
63UASM_L_LA(_copy_pref_both)
64UASM_L_LA(_copy_pref_store)
65
66/* We need one branch and therefore one relocation per target label. */
67static struct uasm_label labels[5];
68static struct uasm_reloc relocs[5];
69
70#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
71#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
72
73/*
74 * R6 has a limited offset of the pref instruction.
75 * Skip it if the offset is more than 9 bits.
76 */
77#define _uasm_i_pref(a, b, c, d) \
78do { \
79 if (cpu_has_mips_r6) { \
80 if (c <= 0xff && c >= -0x100) \
81 uasm_i_pref(a, b, c, d);\
82 } else { \
83 uasm_i_pref(a, b, c, d); \
84 } \
85} while(0)
86
87static int pref_bias_clear_store;
88static int pref_bias_copy_load;
89static int pref_bias_copy_store;
90
91static u32 pref_src_mode;
92static u32 pref_dst_mode;
93
94static int clear_word_size;
95static int copy_word_size;
96
97static int half_clear_loop_size;
98static int half_copy_loop_size;
99
100static int cache_line_size;
101#define cache_line_mask() (cache_line_size - 1)
102
103static inline void
104pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off)
105{
106 if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) {
107 if (off > 0x7fff) {
108 uasm_i_lui(buf, T9, uasm_rel_hi(off));
109 uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off));
110 } else
111 uasm_i_addiu(buf, T9, ZERO, off);
112 uasm_i_daddu(buf, reg1, reg2, T9);
113 } else {
114 if (off > 0x7fff) {
115 uasm_i_lui(buf, T9, uasm_rel_hi(off));
116 uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off));
117 UASM_i_ADDU(buf, reg1, reg2, T9);
118 } else
119 UASM_i_ADDIU(buf, reg1, reg2, off);
120 }
121}
122
123static void set_prefetch_parameters(void)
124{
125 if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg)
126 clear_word_size = 8;
127 else
128 clear_word_size = 4;
129
130 if (cpu_has_64bit_gp_regs)
131 copy_word_size = 8;
132 else
133 copy_word_size = 4;
134
135 /*
136 * The pref's used here are using "streaming" hints, which cause the
137 * copied data to be kicked out of the cache sooner. A page copy often
138 * ends up copying a lot more data than is commonly used, so this seems
139 * to make sense in terms of reducing cache pollution, but I've no real
140 * performance data to back this up.
141 */
142 if (cpu_has_prefetch) {
143 /*
144 * XXX: Most prefetch bias values in here are based on
145 * guesswork.
146 */
147 cache_line_size = cpu_dcache_line_size();
148 switch (current_cpu_type()) {
149 case CPU_R5500:
150 case CPU_TX49XX:
151 /* These processors only support the Pref_Load. */
152 pref_bias_copy_load = 256;
153 break;
154
155 case CPU_R10000:
156 case CPU_R12000:
157 case CPU_R14000:
158 case CPU_R16000:
159 /*
160 * Those values have been experimentally tuned for an
161 * Origin 200.
162 */
163 pref_bias_clear_store = 512;
164 pref_bias_copy_load = 256;
165 pref_bias_copy_store = 256;
166 pref_src_mode = Pref_LoadStreamed;
167 pref_dst_mode = Pref_StoreStreamed;
168 break;
169
170 case CPU_SB1:
171 case CPU_SB1A:
172 pref_bias_clear_store = 128;
173 pref_bias_copy_load = 128;
174 pref_bias_copy_store = 128;
175 /*
176 * SB1 pass1 Pref_LoadStreamed/Pref_StoreStreamed
177 * hints are broken.
178 */
179 if (current_cpu_type() == CPU_SB1 &&
180 (current_cpu_data.processor_id & 0xff) < 0x02) {
181 pref_src_mode = Pref_Load;
182 pref_dst_mode = Pref_Store;
183 } else {
184 pref_src_mode = Pref_LoadStreamed;
185 pref_dst_mode = Pref_StoreStreamed;
186 }
187 break;
188
189 case CPU_LOONGSON64:
190 /* Loongson-3 only support the Pref_Load/Pref_Store. */
191 pref_bias_clear_store = 128;
192 pref_bias_copy_load = 128;
193 pref_bias_copy_store = 128;
194 pref_src_mode = Pref_Load;
195 pref_dst_mode = Pref_Store;
196 break;
197
198 default:
199 pref_bias_clear_store = 128;
200 pref_bias_copy_load = 256;
201 pref_bias_copy_store = 128;
202 pref_src_mode = Pref_LoadStreamed;
203 if (cpu_has_mips_r6)
204 /*
205 * Bit 30 (Pref_PrepareForStore) has been
206 * removed from MIPS R6. Use bit 5
207 * (Pref_StoreStreamed).
208 */
209 pref_dst_mode = Pref_StoreStreamed;
210 else
211 pref_dst_mode = Pref_PrepareForStore;
212 break;
213 }
214 } else {
215 if (cpu_has_cache_cdex_s)
216 cache_line_size = cpu_scache_line_size();
217 else if (cpu_has_cache_cdex_p)
218 cache_line_size = cpu_dcache_line_size();
219 }
220 /*
221 * Too much unrolling will overflow the available space in
222 * clear_space_array / copy_page_array.
223 */
224 half_clear_loop_size = min(16 * clear_word_size,
225 max(cache_line_size >> 1,
226 4 * clear_word_size));
227 half_copy_loop_size = min(16 * copy_word_size,
228 max(cache_line_size >> 1,
229 4 * copy_word_size));
230}
231
232static void build_clear_store(u32 **buf, int off)
233{
234 if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) {
235 uasm_i_sd(buf, ZERO, off, A0);
236 } else {
237 uasm_i_sw(buf, ZERO, off, A0);
238 }
239}
240
241static inline void build_clear_pref(u32 **buf, int off)
242{
243 if (off & cache_line_mask())
244 return;
245
246 if (pref_bias_clear_store) {
247 _uasm_i_pref(buf, pref_dst_mode, pref_bias_clear_store + off,
248 A0);
249 } else if (cache_line_size == (half_clear_loop_size << 1)) {
250 if (cpu_has_cache_cdex_s) {
251 uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0);
252 } else if (cpu_has_cache_cdex_p) {
253 if (IS_ENABLED(CONFIG_WAR_R4600_V1_HIT_CACHEOP) &&
254 cpu_is_r4600_v1_x()) {
255 uasm_i_nop(buf);
256 uasm_i_nop(buf);
257 uasm_i_nop(buf);
258 uasm_i_nop(buf);
259 }
260
261 if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) &&
262 cpu_is_r4600_v2_x())
263 uasm_i_lw(buf, ZERO, ZERO, AT);
264
265 uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0);
266 }
267 }
268}
269
270extern u32 __clear_page_start;
271extern u32 __clear_page_end;
272extern u32 __copy_page_start;
273extern u32 __copy_page_end;
274
275void build_clear_page(void)
276{
277 int off;
278 u32 *buf = &__clear_page_start;
279 struct uasm_label *l = labels;
280 struct uasm_reloc *r = relocs;
281 int i;
282 static atomic_t run_once = ATOMIC_INIT(0);
283
284 if (atomic_xchg(&run_once, 1)) {
285 return;
286 }
287
288 memset(labels, 0, sizeof(labels));
289 memset(relocs, 0, sizeof(relocs));
290
291 set_prefetch_parameters();
292
293 /*
294 * This algorithm makes the following assumptions:
295 * - The prefetch bias is a multiple of 2 words.
296 * - The prefetch bias is less than one page.
297 */
298 BUG_ON(pref_bias_clear_store % (2 * clear_word_size));
299 BUG_ON(PAGE_SIZE < pref_bias_clear_store);
300
301 off = PAGE_SIZE - pref_bias_clear_store;
302 if (off > 0xffff || !pref_bias_clear_store)
303 pg_addiu(&buf, A2, A0, off);
304 else
305 uasm_i_ori(&buf, A2, A0, off);
306
307 if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) && cpu_is_r4600_v2_x())
308 uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000));
309
310 off = cache_line_size ? min(8, pref_bias_clear_store / cache_line_size)
311 * cache_line_size : 0;
312 while (off) {
313 build_clear_pref(&buf, -off);
314 off -= cache_line_size;
315 }
316 uasm_l_clear_pref(&l, buf);
317 do {
318 build_clear_pref(&buf, off);
319 build_clear_store(&buf, off);
320 off += clear_word_size;
321 } while (off < half_clear_loop_size);
322 pg_addiu(&buf, A0, A0, 2 * off);
323 off = -off;
324 do {
325 build_clear_pref(&buf, off);
326 if (off == -clear_word_size)
327 uasm_il_bne(&buf, &r, A0, A2, label_clear_pref);
328 build_clear_store(&buf, off);
329 off += clear_word_size;
330 } while (off < 0);
331
332 if (pref_bias_clear_store) {
333 pg_addiu(&buf, A2, A0, pref_bias_clear_store);
334 uasm_l_clear_nopref(&l, buf);
335 off = 0;
336 do {
337 build_clear_store(&buf, off);
338 off += clear_word_size;
339 } while (off < half_clear_loop_size);
340 pg_addiu(&buf, A0, A0, 2 * off);
341 off = -off;
342 do {
343 if (off == -clear_word_size)
344 uasm_il_bne(&buf, &r, A0, A2,
345 label_clear_nopref);
346 build_clear_store(&buf, off);
347 off += clear_word_size;
348 } while (off < 0);
349 }
350
351 uasm_i_jr(&buf, RA);
352 uasm_i_nop(&buf);
353
354 BUG_ON(buf > &__clear_page_end);
355
356 uasm_resolve_relocs(relocs, labels);
357
358 pr_debug("Synthesized clear page handler (%u instructions).\n",
359 (u32)(buf - &__clear_page_start));
360
361 pr_debug("\t.set push\n");
362 pr_debug("\t.set noreorder\n");
363 for (i = 0; i < (buf - &__clear_page_start); i++)
364 pr_debug("\t.word 0x%08x\n", (&__clear_page_start)[i]);
365 pr_debug("\t.set pop\n");
366}
367
368static void build_copy_load(u32 **buf, int reg, int off)
369{
370 if (cpu_has_64bit_gp_regs) {
371 uasm_i_ld(buf, reg, off, A1);
372 } else {
373 uasm_i_lw(buf, reg, off, A1);
374 }
375}
376
377static void build_copy_store(u32 **buf, int reg, int off)
378{
379 if (cpu_has_64bit_gp_regs) {
380 uasm_i_sd(buf, reg, off, A0);
381 } else {
382 uasm_i_sw(buf, reg, off, A0);
383 }
384}
385
386static inline void build_copy_load_pref(u32 **buf, int off)
387{
388 if (off & cache_line_mask())
389 return;
390
391 if (pref_bias_copy_load)
392 _uasm_i_pref(buf, pref_src_mode, pref_bias_copy_load + off, A1);
393}
394
395static inline void build_copy_store_pref(u32 **buf, int off)
396{
397 if (off & cache_line_mask())
398 return;
399
400 if (pref_bias_copy_store) {
401 _uasm_i_pref(buf, pref_dst_mode, pref_bias_copy_store + off,
402 A0);
403 } else if (cache_line_size == (half_copy_loop_size << 1)) {
404 if (cpu_has_cache_cdex_s) {
405 uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0);
406 } else if (cpu_has_cache_cdex_p) {
407 if (IS_ENABLED(CONFIG_WAR_R4600_V1_HIT_CACHEOP) &&
408 cpu_is_r4600_v1_x()) {
409 uasm_i_nop(buf);
410 uasm_i_nop(buf);
411 uasm_i_nop(buf);
412 uasm_i_nop(buf);
413 }
414
415 if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) &&
416 cpu_is_r4600_v2_x())
417 uasm_i_lw(buf, ZERO, ZERO, AT);
418
419 uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0);
420 }
421 }
422}
423
424void build_copy_page(void)
425{
426 int off;
427 u32 *buf = &__copy_page_start;
428 struct uasm_label *l = labels;
429 struct uasm_reloc *r = relocs;
430 int i;
431 static atomic_t run_once = ATOMIC_INIT(0);
432
433 if (atomic_xchg(&run_once, 1)) {
434 return;
435 }
436
437 memset(labels, 0, sizeof(labels));
438 memset(relocs, 0, sizeof(relocs));
439
440 set_prefetch_parameters();
441
442 /*
443 * This algorithm makes the following assumptions:
444 * - All prefetch biases are multiples of 8 words.
445 * - The prefetch biases are less than one page.
446 * - The store prefetch bias isn't greater than the load
447 * prefetch bias.
448 */
449 BUG_ON(pref_bias_copy_load % (8 * copy_word_size));
450 BUG_ON(pref_bias_copy_store % (8 * copy_word_size));
451 BUG_ON(PAGE_SIZE < pref_bias_copy_load);
452 BUG_ON(pref_bias_copy_store > pref_bias_copy_load);
453
454 off = PAGE_SIZE - pref_bias_copy_load;
455 if (off > 0xffff || !pref_bias_copy_load)
456 pg_addiu(&buf, A2, A0, off);
457 else
458 uasm_i_ori(&buf, A2, A0, off);
459
460 if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) && cpu_is_r4600_v2_x())
461 uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000));
462
463 off = cache_line_size ? min(8, pref_bias_copy_load / cache_line_size) *
464 cache_line_size : 0;
465 while (off) {
466 build_copy_load_pref(&buf, -off);
467 off -= cache_line_size;
468 }
469 off = cache_line_size ? min(8, pref_bias_copy_store / cache_line_size) *
470 cache_line_size : 0;
471 while (off) {
472 build_copy_store_pref(&buf, -off);
473 off -= cache_line_size;
474 }
475 uasm_l_copy_pref_both(&l, buf);
476 do {
477 build_copy_load_pref(&buf, off);
478 build_copy_load(&buf, T0, off);
479 build_copy_load_pref(&buf, off + copy_word_size);
480 build_copy_load(&buf, T1, off + copy_word_size);
481 build_copy_load_pref(&buf, off + 2 * copy_word_size);
482 build_copy_load(&buf, T2, off + 2 * copy_word_size);
483 build_copy_load_pref(&buf, off + 3 * copy_word_size);
484 build_copy_load(&buf, T3, off + 3 * copy_word_size);
485 build_copy_store_pref(&buf, off);
486 build_copy_store(&buf, T0, off);
487 build_copy_store_pref(&buf, off + copy_word_size);
488 build_copy_store(&buf, T1, off + copy_word_size);
489 build_copy_store_pref(&buf, off + 2 * copy_word_size);
490 build_copy_store(&buf, T2, off + 2 * copy_word_size);
491 build_copy_store_pref(&buf, off + 3 * copy_word_size);
492 build_copy_store(&buf, T3, off + 3 * copy_word_size);
493 off += 4 * copy_word_size;
494 } while (off < half_copy_loop_size);
495 pg_addiu(&buf, A1, A1, 2 * off);
496 pg_addiu(&buf, A0, A0, 2 * off);
497 off = -off;
498 do {
499 build_copy_load_pref(&buf, off);
500 build_copy_load(&buf, T0, off);
501 build_copy_load_pref(&buf, off + copy_word_size);
502 build_copy_load(&buf, T1, off + copy_word_size);
503 build_copy_load_pref(&buf, off + 2 * copy_word_size);
504 build_copy_load(&buf, T2, off + 2 * copy_word_size);
505 build_copy_load_pref(&buf, off + 3 * copy_word_size);
506 build_copy_load(&buf, T3, off + 3 * copy_word_size);
507 build_copy_store_pref(&buf, off);
508 build_copy_store(&buf, T0, off);
509 build_copy_store_pref(&buf, off + copy_word_size);
510 build_copy_store(&buf, T1, off + copy_word_size);
511 build_copy_store_pref(&buf, off + 2 * copy_word_size);
512 build_copy_store(&buf, T2, off + 2 * copy_word_size);
513 build_copy_store_pref(&buf, off + 3 * copy_word_size);
514 if (off == -(4 * copy_word_size))
515 uasm_il_bne(&buf, &r, A2, A0, label_copy_pref_both);
516 build_copy_store(&buf, T3, off + 3 * copy_word_size);
517 off += 4 * copy_word_size;
518 } while (off < 0);
519
520 if (pref_bias_copy_load - pref_bias_copy_store) {
521 pg_addiu(&buf, A2, A0,
522 pref_bias_copy_load - pref_bias_copy_store);
523 uasm_l_copy_pref_store(&l, buf);
524 off = 0;
525 do {
526 build_copy_load(&buf, T0, off);
527 build_copy_load(&buf, T1, off + copy_word_size);
528 build_copy_load(&buf, T2, off + 2 * copy_word_size);
529 build_copy_load(&buf, T3, off + 3 * copy_word_size);
530 build_copy_store_pref(&buf, off);
531 build_copy_store(&buf, T0, off);
532 build_copy_store_pref(&buf, off + copy_word_size);
533 build_copy_store(&buf, T1, off + copy_word_size);
534 build_copy_store_pref(&buf, off + 2 * copy_word_size);
535 build_copy_store(&buf, T2, off + 2 * copy_word_size);
536 build_copy_store_pref(&buf, off + 3 * copy_word_size);
537 build_copy_store(&buf, T3, off + 3 * copy_word_size);
538 off += 4 * copy_word_size;
539 } while (off < half_copy_loop_size);
540 pg_addiu(&buf, A1, A1, 2 * off);
541 pg_addiu(&buf, A0, A0, 2 * off);
542 off = -off;
543 do {
544 build_copy_load(&buf, T0, off);
545 build_copy_load(&buf, T1, off + copy_word_size);
546 build_copy_load(&buf, T2, off + 2 * copy_word_size);
547 build_copy_load(&buf, T3, off + 3 * copy_word_size);
548 build_copy_store_pref(&buf, off);
549 build_copy_store(&buf, T0, off);
550 build_copy_store_pref(&buf, off + copy_word_size);
551 build_copy_store(&buf, T1, off + copy_word_size);
552 build_copy_store_pref(&buf, off + 2 * copy_word_size);
553 build_copy_store(&buf, T2, off + 2 * copy_word_size);
554 build_copy_store_pref(&buf, off + 3 * copy_word_size);
555 if (off == -(4 * copy_word_size))
556 uasm_il_bne(&buf, &r, A2, A0,
557 label_copy_pref_store);
558 build_copy_store(&buf, T3, off + 3 * copy_word_size);
559 off += 4 * copy_word_size;
560 } while (off < 0);
561 }
562
563 if (pref_bias_copy_store) {
564 pg_addiu(&buf, A2, A0, pref_bias_copy_store);
565 uasm_l_copy_nopref(&l, buf);
566 off = 0;
567 do {
568 build_copy_load(&buf, T0, off);
569 build_copy_load(&buf, T1, off + copy_word_size);
570 build_copy_load(&buf, T2, off + 2 * copy_word_size);
571 build_copy_load(&buf, T3, off + 3 * copy_word_size);
572 build_copy_store(&buf, T0, off);
573 build_copy_store(&buf, T1, off + copy_word_size);
574 build_copy_store(&buf, T2, off + 2 * copy_word_size);
575 build_copy_store(&buf, T3, off + 3 * copy_word_size);
576 off += 4 * copy_word_size;
577 } while (off < half_copy_loop_size);
578 pg_addiu(&buf, A1, A1, 2 * off);
579 pg_addiu(&buf, A0, A0, 2 * off);
580 off = -off;
581 do {
582 build_copy_load(&buf, T0, off);
583 build_copy_load(&buf, T1, off + copy_word_size);
584 build_copy_load(&buf, T2, off + 2 * copy_word_size);
585 build_copy_load(&buf, T3, off + 3 * copy_word_size);
586 build_copy_store(&buf, T0, off);
587 build_copy_store(&buf, T1, off + copy_word_size);
588 build_copy_store(&buf, T2, off + 2 * copy_word_size);
589 if (off == -(4 * copy_word_size))
590 uasm_il_bne(&buf, &r, A2, A0,
591 label_copy_nopref);
592 build_copy_store(&buf, T3, off + 3 * copy_word_size);
593 off += 4 * copy_word_size;
594 } while (off < 0);
595 }
596
597 uasm_i_jr(&buf, RA);
598 uasm_i_nop(&buf);
599
600 BUG_ON(buf > &__copy_page_end);
601
602 uasm_resolve_relocs(relocs, labels);
603
604 pr_debug("Synthesized copy page handler (%u instructions).\n",
605 (u32)(buf - &__copy_page_start));
606
607 pr_debug("\t.set push\n");
608 pr_debug("\t.set noreorder\n");
609 for (i = 0; i < (buf - &__copy_page_start); i++)
610 pr_debug("\t.word 0x%08x\n", (&__copy_page_start)[i]);
611 pr_debug("\t.set pop\n");
612}
613
614#ifdef CONFIG_SIBYTE_DMA_PAGEOPS
615extern void clear_page_cpu(void *page);
616extern void copy_page_cpu(void *to, void *from);
617
618/*
619 * Pad descriptors to cacheline, since each is exclusively owned by a
620 * particular CPU.
621 */
622struct dmadscr {
623 u64 dscr_a;
624 u64 dscr_b;
625 u64 pad_a;
626 u64 pad_b;
627} ____cacheline_aligned_in_smp page_descr[DM_NUM_CHANNELS];
628
629void clear_page(void *page)
630{
631 u64 to_phys = CPHYSADDR((unsigned long)page);
632 unsigned int cpu = smp_processor_id();
633
634 /* if the page is not in KSEG0, use old way */
635 if ((long)KSEGX((unsigned long)page) != (long)CKSEG0)
636 return clear_page_cpu(page);
637
638 page_descr[cpu].dscr_a = to_phys | M_DM_DSCRA_ZERO_MEM |
639 M_DM_DSCRA_L2C_DEST | M_DM_DSCRA_INTERRUPT;
640 page_descr[cpu].dscr_b = V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE);
641 __raw_writeq(1, IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT)));
642
643 /*
644 * Don't really want to do it this way, but there's no
645 * reliable way to delay completion detection.
646 */
647 while (!(__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG)))
648 & M_DM_DSCR_BASE_INTERRUPT))
649 ;
650 __raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
651}
652EXPORT_SYMBOL(clear_page);
653
654void copy_page(void *to, void *from)
655{
656 u64 from_phys = CPHYSADDR((unsigned long)from);
657 u64 to_phys = CPHYSADDR((unsigned long)to);
658 unsigned int cpu = smp_processor_id();
659
660 /* if any page is not in KSEG0, use old way */
661 if ((long)KSEGX((unsigned long)to) != (long)CKSEG0
662 || (long)KSEGX((unsigned long)from) != (long)CKSEG0)
663 return copy_page_cpu(to, from);
664
665 page_descr[cpu].dscr_a = to_phys | M_DM_DSCRA_L2C_DEST |
666 M_DM_DSCRA_INTERRUPT;
667 page_descr[cpu].dscr_b = from_phys | V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE);
668 __raw_writeq(1, IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT)));
669
670 /*
671 * Don't really want to do it this way, but there's no
672 * reliable way to delay completion detection.
673 */
674 while (!(__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG)))
675 & M_DM_DSCR_BASE_INTERRUPT))
676 ;
677 __raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE)));
678}
679EXPORT_SYMBOL(copy_page);
680
681#endif /* CONFIG_SIBYTE_DMA_PAGEOPS */
diff --git a/arch/mips/mm/pgtable-32.c b/arch/mips/mm/pgtable-32.c
new file mode 100644
index 000000000..bd4b0656a
--- /dev/null
+++ b/arch/mips/mm/pgtable-32.c
@@ -0,0 +1,91 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2003 by Ralf Baechle
7 */
8#include <linux/init.h>
9#include <linux/mm.h>
10#include <linux/memblock.h>
11#include <linux/highmem.h>
12#include <asm/fixmap.h>
13#include <asm/pgalloc.h>
14#include <asm/tlbflush.h>
15
16void pgd_init(unsigned long page)
17{
18 unsigned long *p = (unsigned long *) page;
19 int i;
20
21 for (i = 0; i < USER_PTRS_PER_PGD; i+=8) {
22 p[i + 0] = (unsigned long) invalid_pte_table;
23 p[i + 1] = (unsigned long) invalid_pte_table;
24 p[i + 2] = (unsigned long) invalid_pte_table;
25 p[i + 3] = (unsigned long) invalid_pte_table;
26 p[i + 4] = (unsigned long) invalid_pte_table;
27 p[i + 5] = (unsigned long) invalid_pte_table;
28 p[i + 6] = (unsigned long) invalid_pte_table;
29 p[i + 7] = (unsigned long) invalid_pte_table;
30 }
31}
32
33#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
34pmd_t mk_pmd(struct page *page, pgprot_t prot)
35{
36 pmd_t pmd;
37
38 pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot);
39
40 return pmd;
41}
42
43
44void set_pmd_at(struct mm_struct *mm, unsigned long addr,
45 pmd_t *pmdp, pmd_t pmd)
46{
47 *pmdp = pmd;
48 flush_tlb_all();
49}
50#endif /* defined(CONFIG_TRANSPARENT_HUGEPAGE) */
51
52void __init pagetable_init(void)
53{
54 unsigned long vaddr;
55 pgd_t *pgd_base;
56#ifdef CONFIG_HIGHMEM
57 pgd_t *pgd;
58 p4d_t *p4d;
59 pud_t *pud;
60 pmd_t *pmd;
61 pte_t *pte;
62#endif
63
64 /* Initialize the entire pgd. */
65 pgd_init((unsigned long)swapper_pg_dir);
66 pgd_init((unsigned long)swapper_pg_dir
67 + sizeof(pgd_t) * USER_PTRS_PER_PGD);
68
69 pgd_base = swapper_pg_dir;
70
71 /*
72 * Fixed mappings:
73 */
74 vaddr = __fix_to_virt(__end_of_fixed_addresses - 1);
75 fixrange_init(vaddr & PMD_MASK, vaddr + FIXADDR_SIZE, pgd_base);
76
77#ifdef CONFIG_HIGHMEM
78 /*
79 * Permanent kmaps:
80 */
81 vaddr = PKMAP_BASE;
82 fixrange_init(vaddr & PMD_MASK, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);
83
84 pgd = swapper_pg_dir + pgd_index(vaddr);
85 p4d = p4d_offset(pgd, vaddr);
86 pud = pud_offset(p4d, vaddr);
87 pmd = pmd_offset(pud, vaddr);
88 pte = pte_offset_kernel(pmd, vaddr);
89 pkmap_page_table = pte;
90#endif
91}
diff --git a/arch/mips/mm/pgtable-64.c b/arch/mips/mm/pgtable-64.c
new file mode 100644
index 000000000..183ff9f9c
--- /dev/null
+++ b/arch/mips/mm/pgtable-64.c
@@ -0,0 +1,125 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1999, 2000 by Silicon Graphics
7 * Copyright (C) 2003 by Ralf Baechle
8 */
9#include <linux/export.h>
10#include <linux/init.h>
11#include <linux/mm.h>
12#include <asm/fixmap.h>
13#include <asm/pgalloc.h>
14#include <asm/tlbflush.h>
15
16void pgd_init(unsigned long page)
17{
18 unsigned long *p, *end;
19 unsigned long entry;
20
21#if !defined(__PAGETABLE_PUD_FOLDED)
22 entry = (unsigned long)invalid_pud_table;
23#elif !defined(__PAGETABLE_PMD_FOLDED)
24 entry = (unsigned long)invalid_pmd_table;
25#else
26 entry = (unsigned long)invalid_pte_table;
27#endif
28
29 p = (unsigned long *) page;
30 end = p + PTRS_PER_PGD;
31
32 do {
33 p[0] = entry;
34 p[1] = entry;
35 p[2] = entry;
36 p[3] = entry;
37 p[4] = entry;
38 p += 8;
39 p[-3] = entry;
40 p[-2] = entry;
41 p[-1] = entry;
42 } while (p != end);
43}
44
45#ifndef __PAGETABLE_PMD_FOLDED
46void pmd_init(unsigned long addr, unsigned long pagetable)
47{
48 unsigned long *p, *end;
49
50 p = (unsigned long *) addr;
51 end = p + PTRS_PER_PMD;
52
53 do {
54 p[0] = pagetable;
55 p[1] = pagetable;
56 p[2] = pagetable;
57 p[3] = pagetable;
58 p[4] = pagetable;
59 p += 8;
60 p[-3] = pagetable;
61 p[-2] = pagetable;
62 p[-1] = pagetable;
63 } while (p != end);
64}
65EXPORT_SYMBOL_GPL(pmd_init);
66#endif
67
68#ifndef __PAGETABLE_PUD_FOLDED
69void pud_init(unsigned long addr, unsigned long pagetable)
70{
71 unsigned long *p, *end;
72
73 p = (unsigned long *)addr;
74 end = p + PTRS_PER_PUD;
75
76 do {
77 p[0] = pagetable;
78 p[1] = pagetable;
79 p[2] = pagetable;
80 p[3] = pagetable;
81 p[4] = pagetable;
82 p += 8;
83 p[-3] = pagetable;
84 p[-2] = pagetable;
85 p[-1] = pagetable;
86 } while (p != end);
87}
88#endif
89
90pmd_t mk_pmd(struct page *page, pgprot_t prot)
91{
92 pmd_t pmd;
93
94 pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot);
95
96 return pmd;
97}
98
99void set_pmd_at(struct mm_struct *mm, unsigned long addr,
100 pmd_t *pmdp, pmd_t pmd)
101{
102 *pmdp = pmd;
103 flush_tlb_all();
104}
105
106void __init pagetable_init(void)
107{
108 unsigned long vaddr;
109 pgd_t *pgd_base;
110
111 /* Initialize the entire pgd. */
112 pgd_init((unsigned long)swapper_pg_dir);
113#ifndef __PAGETABLE_PUD_FOLDED
114 pud_init((unsigned long)invalid_pud_table, (unsigned long)invalid_pmd_table);
115#endif
116#ifndef __PAGETABLE_PMD_FOLDED
117 pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table);
118#endif
119 pgd_base = swapper_pg_dir;
120 /*
121 * Fixed mappings:
122 */
123 vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
124 fixrange_init(vaddr, vaddr + FIXADDR_SIZE, pgd_base);
125}
diff --git a/arch/mips/mm/pgtable.c b/arch/mips/mm/pgtable.c
new file mode 100644
index 000000000..05560b042
--- /dev/null
+++ b/arch/mips/mm/pgtable.c
@@ -0,0 +1,25 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 */
6#include <linux/export.h>
7#include <linux/mm.h>
8#include <linux/string.h>
9#include <asm/pgalloc.h>
10
11pgd_t *pgd_alloc(struct mm_struct *mm)
12{
13 pgd_t *ret, *init;
14
15 ret = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_ORDER);
16 if (ret) {
17 init = pgd_offset(&init_mm, 0UL);
18 pgd_init((unsigned long)ret);
19 memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
20 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
21 }
22
23 return ret;
24}
25EXPORT_SYMBOL_GPL(pgd_alloc);
diff --git a/arch/mips/mm/sc-debugfs.c b/arch/mips/mm/sc-debugfs.c
new file mode 100644
index 000000000..80ff39471
--- /dev/null
+++ b/arch/mips/mm/sc-debugfs.c
@@ -0,0 +1,61 @@
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2015 Imagination Technologies
4 * Author: Paul Burton <paul.burton@mips.com>
5 */
6
7#include <asm/bcache.h>
8#include <asm/debug.h>
9#include <linux/uaccess.h>
10#include <linux/debugfs.h>
11#include <linux/init.h>
12
13static ssize_t sc_prefetch_read(struct file *file, char __user *user_buf,
14 size_t count, loff_t *ppos)
15{
16 bool enabled = bc_prefetch_is_enabled();
17 char buf[3];
18
19 buf[0] = enabled ? 'Y' : 'N';
20 buf[1] = '\n';
21 buf[2] = 0;
22
23 return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
24}
25
26static ssize_t sc_prefetch_write(struct file *file,
27 const char __user *user_buf,
28 size_t count, loff_t *ppos)
29{
30 bool enabled;
31 int err;
32
33 err = kstrtobool_from_user(user_buf, count, &enabled);
34 if (err)
35 return err;
36
37 if (enabled)
38 bc_prefetch_enable();
39 else
40 bc_prefetch_disable();
41
42 return count;
43}
44
45static const struct file_operations sc_prefetch_fops = {
46 .open = simple_open,
47 .llseek = default_llseek,
48 .read = sc_prefetch_read,
49 .write = sc_prefetch_write,
50};
51
52static int __init sc_debugfs_init(void)
53{
54 struct dentry *dir;
55
56 dir = debugfs_create_dir("l2cache", mips_debugfs_dir);
57 debugfs_create_file("prefetch", S_IRUGO | S_IWUSR, dir, NULL,
58 &sc_prefetch_fops);
59 return 0;
60}
61late_initcall(sc_debugfs_init);
diff --git a/arch/mips/mm/sc-ip22.c b/arch/mips/mm/sc-ip22.c
new file mode 100644
index 000000000..d7238687d
--- /dev/null
+++ b/arch/mips/mm/sc-ip22.c
@@ -0,0 +1,190 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * sc-ip22.c: Indy cache management functions.
4 *
5 * Copyright (C) 1997, 2001 Ralf Baechle (ralf@gnu.org),
6 * derived from r4xx0.c by David S. Miller (davem@davemloft.net).
7 */
8#include <linux/init.h>
9#include <linux/kernel.h>
10#include <linux/sched.h>
11#include <linux/mm.h>
12
13#include <asm/bcache.h>
14#include <asm/page.h>
15#include <asm/bootinfo.h>
16#include <asm/sgi/ip22.h>
17#include <asm/sgi/mc.h>
18
19/* Secondary cache size in bytes, if present. */
20static unsigned long scache_size;
21
22#undef DEBUG_CACHE
23
24#define SC_SIZE 0x00080000
25#define SC_LINE 32
26#define CI_MASK (SC_SIZE - SC_LINE)
27#define SC_INDEX(n) ((n) & CI_MASK)
28
29static inline void indy_sc_wipe(unsigned long first, unsigned long last)
30{
31 unsigned long tmp;
32
33 __asm__ __volatile__(
34 " .set push # indy_sc_wipe \n"
35 " .set noreorder \n"
36 " .set mips3 \n"
37 " .set noat \n"
38 " mfc0 %2, $12 \n"
39 " li $1, 0x80 # Go 64 bit \n"
40 " mtc0 $1, $12 \n"
41 " \n"
42 " # \n"
43 " # Open code a dli $1, 0x9000000080000000 \n"
44 " # \n"
45 " # Required because binutils 2.25 will happily accept \n"
46 " # 64 bit instructions in .set mips3 mode but puke on \n"
47 " # 64 bit constants when generating 32 bit ELF \n"
48 " # \n"
49 " lui $1,0x9000 \n"
50 " dsll $1,$1,0x10 \n"
51 " ori $1,$1,0x8000 \n"
52 " dsll $1,$1,0x10 \n"
53 " \n"
54 " or %0, $1 # first line to flush \n"
55 " or %1, $1 # last line to flush \n"
56 " .set at \n"
57 " \n"
58 "1: sw $0, 0(%0) \n"
59 " bne %0, %1, 1b \n"
60 " daddu %0, 32 \n"
61 " \n"
62 " mtc0 %2, $12 # Back to 32 bit \n"
63 " nop # pipeline hazard \n"
64 " nop \n"
65 " nop \n"
66 " nop \n"
67 " .set pop \n"
68 : "=r" (first), "=r" (last), "=&r" (tmp)
69 : "0" (first), "1" (last));
70}
71
72static void indy_sc_wback_invalidate(unsigned long addr, unsigned long size)
73{
74 unsigned long first_line, last_line;
75 unsigned long flags;
76
77#ifdef DEBUG_CACHE
78 printk("indy_sc_wback_invalidate[%08lx,%08lx]", addr, size);
79#endif
80
81 /* Catch bad driver code */
82 BUG_ON(size == 0);
83
84 /* Which lines to flush? */
85 first_line = SC_INDEX(addr);
86 last_line = SC_INDEX(addr + size - 1);
87
88 local_irq_save(flags);
89 if (first_line <= last_line) {
90 indy_sc_wipe(first_line, last_line);
91 goto out;
92 }
93
94 indy_sc_wipe(first_line, SC_SIZE - SC_LINE);
95 indy_sc_wipe(0, last_line);
96out:
97 local_irq_restore(flags);
98}
99
100static void indy_sc_enable(void)
101{
102 unsigned long addr, tmp1, tmp2;
103
104 /* This is really cool... */
105#ifdef DEBUG_CACHE
106 printk("Enabling R4600 SCACHE\n");
107#endif
108 __asm__ __volatile__(
109 ".set\tpush\n\t"
110 ".set\tnoreorder\n\t"
111 ".set\tmips3\n\t"
112 "mfc0\t%2, $12\n\t"
113 "nop; nop; nop; nop;\n\t"
114 "li\t%1, 0x80\n\t"
115 "mtc0\t%1, $12\n\t"
116 "nop; nop; nop; nop;\n\t"
117 "li\t%0, 0x1\n\t"
118 "dsll\t%0, 31\n\t"
119 "lui\t%1, 0x9000\n\t"
120 "dsll32\t%1, 0\n\t"
121 "or\t%0, %1, %0\n\t"
122 "sb\t$0, 0(%0)\n\t"
123 "mtc0\t$0, $12\n\t"
124 "nop; nop; nop; nop;\n\t"
125 "mtc0\t%2, $12\n\t"
126 "nop; nop; nop; nop;\n\t"
127 ".set\tpop"
128 : "=r" (tmp1), "=r" (tmp2), "=r" (addr));
129}
130
131static void indy_sc_disable(void)
132{
133 unsigned long tmp1, tmp2, tmp3;
134
135#ifdef DEBUG_CACHE
136 printk("Disabling R4600 SCACHE\n");
137#endif
138 __asm__ __volatile__(
139 ".set\tpush\n\t"
140 ".set\tnoreorder\n\t"
141 ".set\tmips3\n\t"
142 "li\t%0, 0x1\n\t"
143 "dsll\t%0, 31\n\t"
144 "lui\t%1, 0x9000\n\t"
145 "dsll32\t%1, 0\n\t"
146 "or\t%0, %1, %0\n\t"
147 "mfc0\t%2, $12\n\t"
148 "nop; nop; nop; nop\n\t"
149 "li\t%1, 0x80\n\t"
150 "mtc0\t%1, $12\n\t"
151 "nop; nop; nop; nop\n\t"
152 "sh\t$0, 0(%0)\n\t"
153 "mtc0\t$0, $12\n\t"
154 "nop; nop; nop; nop\n\t"
155 "mtc0\t%2, $12\n\t"
156 "nop; nop; nop; nop\n\t"
157 ".set\tpop"
158 : "=r" (tmp1), "=r" (tmp2), "=r" (tmp3));
159}
160
161static inline int __init indy_sc_probe(void)
162{
163 unsigned int size = ip22_eeprom_read(&sgimc->eeprom, 17);
164 if (size == 0)
165 return 0;
166
167 size <<= PAGE_SHIFT;
168 printk(KERN_INFO "R4600/R5000 SCACHE size %dK, linesize 32 bytes.\n",
169 size >> 10);
170 scache_size = size;
171
172 return 1;
173}
174
175/* XXX Check with wje if the Indy caches can differentiate between
176 writeback + invalidate and just invalidate. */
177static struct bcache_ops indy_sc_ops = {
178 .bc_enable = indy_sc_enable,
179 .bc_disable = indy_sc_disable,
180 .bc_wback_inv = indy_sc_wback_invalidate,
181 .bc_inv = indy_sc_wback_invalidate
182};
183
184void indy_sc_init(void)
185{
186 if (indy_sc_probe()) {
187 indy_sc_enable();
188 bcops = &indy_sc_ops;
189 }
190}
diff --git a/arch/mips/mm/sc-mips.c b/arch/mips/mm/sc-mips.c
new file mode 100644
index 000000000..06ec304ad
--- /dev/null
+++ b/arch/mips/mm/sc-mips.c
@@ -0,0 +1,264 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2006 Chris Dearman (chris@mips.com),
4 */
5#include <linux/init.h>
6#include <linux/kernel.h>
7#include <linux/sched.h>
8#include <linux/mm.h>
9
10#include <asm/cpu-type.h>
11#include <asm/mipsregs.h>
12#include <asm/bcache.h>
13#include <asm/cacheops.h>
14#include <asm/page.h>
15#include <asm/mmu_context.h>
16#include <asm/r4kcache.h>
17#include <asm/mips-cps.h>
18#include <asm/bootinfo.h>
19
20/*
21 * MIPS32/MIPS64 L2 cache handling
22 */
23
24/*
25 * Writeback and invalidate the secondary cache before DMA.
26 */
27static void mips_sc_wback_inv(unsigned long addr, unsigned long size)
28{
29 blast_scache_range(addr, addr + size);
30}
31
32/*
33 * Invalidate the secondary cache before DMA.
34 */
35static void mips_sc_inv(unsigned long addr, unsigned long size)
36{
37 unsigned long lsize = cpu_scache_line_size();
38 unsigned long almask = ~(lsize - 1);
39
40 cache_op(Hit_Writeback_Inv_SD, addr & almask);
41 cache_op(Hit_Writeback_Inv_SD, (addr + size - 1) & almask);
42 blast_inv_scache_range(addr, addr + size);
43}
44
45static void mips_sc_enable(void)
46{
47 /* L2 cache is permanently enabled */
48}
49
50static void mips_sc_disable(void)
51{
52 /* L2 cache is permanently enabled */
53}
54
55static void mips_sc_prefetch_enable(void)
56{
57 unsigned long pftctl;
58
59 if (mips_cm_revision() < CM_REV_CM2_5)
60 return;
61
62 /*
63 * If there is one or more L2 prefetch unit present then enable
64 * prefetching for both code & data, for all ports.
65 */
66 pftctl = read_gcr_l2_pft_control();
67 if (pftctl & CM_GCR_L2_PFT_CONTROL_NPFT) {
68 pftctl &= ~CM_GCR_L2_PFT_CONTROL_PAGEMASK;
69 pftctl |= PAGE_MASK & CM_GCR_L2_PFT_CONTROL_PAGEMASK;
70 pftctl |= CM_GCR_L2_PFT_CONTROL_PFTEN;
71 write_gcr_l2_pft_control(pftctl);
72
73 set_gcr_l2_pft_control_b(CM_GCR_L2_PFT_CONTROL_B_PORTID |
74 CM_GCR_L2_PFT_CONTROL_B_CEN);
75 }
76}
77
78static void mips_sc_prefetch_disable(void)
79{
80 if (mips_cm_revision() < CM_REV_CM2_5)
81 return;
82
83 clear_gcr_l2_pft_control(CM_GCR_L2_PFT_CONTROL_PFTEN);
84 clear_gcr_l2_pft_control_b(CM_GCR_L2_PFT_CONTROL_B_PORTID |
85 CM_GCR_L2_PFT_CONTROL_B_CEN);
86}
87
88static bool mips_sc_prefetch_is_enabled(void)
89{
90 unsigned long pftctl;
91
92 if (mips_cm_revision() < CM_REV_CM2_5)
93 return false;
94
95 pftctl = read_gcr_l2_pft_control();
96 if (!(pftctl & CM_GCR_L2_PFT_CONTROL_NPFT))
97 return false;
98 return !!(pftctl & CM_GCR_L2_PFT_CONTROL_PFTEN);
99}
100
101static struct bcache_ops mips_sc_ops = {
102 .bc_enable = mips_sc_enable,
103 .bc_disable = mips_sc_disable,
104 .bc_wback_inv = mips_sc_wback_inv,
105 .bc_inv = mips_sc_inv,
106 .bc_prefetch_enable = mips_sc_prefetch_enable,
107 .bc_prefetch_disable = mips_sc_prefetch_disable,
108 .bc_prefetch_is_enabled = mips_sc_prefetch_is_enabled,
109};
110
111/*
112 * Check if the L2 cache controller is activated on a particular platform.
113 * MTI's L2 controller and the L2 cache controller of Broadcom's BMIPS
114 * cores both use c0_config2's bit 12 as "L2 Bypass" bit, that is the
115 * cache being disabled. However there is no guarantee for this to be
116 * true on all platforms. In an act of stupidity the spec defined bits
117 * 12..15 as implementation defined so below function will eventually have
118 * to be replaced by a platform specific probe.
119 */
120static inline int mips_sc_is_activated(struct cpuinfo_mips *c)
121{
122 unsigned int config2 = read_c0_config2();
123 unsigned int tmp;
124
125 /* Check the bypass bit (L2B) */
126 switch (current_cpu_type()) {
127 case CPU_34K:
128 case CPU_74K:
129 case CPU_1004K:
130 case CPU_1074K:
131 case CPU_INTERAPTIV:
132 case CPU_PROAPTIV:
133 case CPU_P5600:
134 case CPU_BMIPS5000:
135 case CPU_QEMU_GENERIC:
136 case CPU_P6600:
137 if (config2 & (1 << 12))
138 return 0;
139 }
140
141 tmp = (config2 >> 4) & 0x0f;
142 if (0 < tmp && tmp <= 7)
143 c->scache.linesz = 2 << tmp;
144 else
145 return 0;
146 return 1;
147}
148
149static int mips_sc_probe_cm3(void)
150{
151 struct cpuinfo_mips *c = &current_cpu_data;
152 unsigned long cfg = read_gcr_l2_config();
153 unsigned long sets, line_sz, assoc;
154
155 if (cfg & CM_GCR_L2_CONFIG_BYPASS)
156 return 0;
157
158 sets = cfg & CM_GCR_L2_CONFIG_SET_SIZE;
159 sets >>= __ffs(CM_GCR_L2_CONFIG_SET_SIZE);
160 if (sets)
161 c->scache.sets = 64 << sets;
162
163 line_sz = cfg & CM_GCR_L2_CONFIG_LINE_SIZE;
164 line_sz >>= __ffs(CM_GCR_L2_CONFIG_LINE_SIZE);
165 if (line_sz)
166 c->scache.linesz = 2 << line_sz;
167
168 assoc = cfg & CM_GCR_L2_CONFIG_ASSOC;
169 assoc >>= __ffs(CM_GCR_L2_CONFIG_ASSOC);
170 c->scache.ways = assoc + 1;
171 c->scache.waysize = c->scache.sets * c->scache.linesz;
172 c->scache.waybit = __ffs(c->scache.waysize);
173
174 if (c->scache.linesz) {
175 c->scache.flags &= ~MIPS_CACHE_NOT_PRESENT;
176 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
177 return 1;
178 }
179
180 return 0;
181}
182
183static inline int mips_sc_probe(void)
184{
185 struct cpuinfo_mips *c = &current_cpu_data;
186 unsigned int config1, config2;
187 unsigned int tmp;
188
189 /* Mark as not present until probe completed */
190 c->scache.flags |= MIPS_CACHE_NOT_PRESENT;
191
192 if (mips_cm_revision() >= CM_REV_CM3)
193 return mips_sc_probe_cm3();
194
195 /* Ignore anything but MIPSxx processors */
196 if (!(c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 |
197 MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 |
198 MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 |
199 MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)))
200 return 0;
201
202 /* Does this MIPS32/MIPS64 CPU have a config2 register? */
203 config1 = read_c0_config1();
204 if (!(config1 & MIPS_CONF_M))
205 return 0;
206
207 config2 = read_c0_config2();
208
209 if (!mips_sc_is_activated(c))
210 return 0;
211
212 tmp = (config2 >> 8) & 0x0f;
213 if (tmp <= 7)
214 c->scache.sets = 64 << tmp;
215 else
216 return 0;
217
218 tmp = (config2 >> 0) & 0x0f;
219 if (tmp <= 7)
220 c->scache.ways = tmp + 1;
221 else
222 return 0;
223
224 if (current_cpu_type() == CPU_XBURST) {
225 switch (mips_machtype) {
226 /*
227 * According to config2 it would be 5-ways, but that is
228 * contradicted by all documentation.
229 */
230 case MACH_INGENIC_JZ4770:
231 case MACH_INGENIC_JZ4775:
232 c->scache.ways = 4;
233 break;
234
235 /*
236 * According to config2 it would be 5-ways and 512-sets,
237 * but that is contradicted by all documentation.
238 */
239 case MACH_INGENIC_X1000:
240 case MACH_INGENIC_X1000E:
241 c->scache.sets = 256;
242 c->scache.ways = 4;
243 break;
244 }
245 }
246
247 c->scache.waysize = c->scache.sets * c->scache.linesz;
248 c->scache.waybit = __ffs(c->scache.waysize);
249
250 c->scache.flags &= ~MIPS_CACHE_NOT_PRESENT;
251
252 return 1;
253}
254
255int mips_sc_init(void)
256{
257 int found = mips_sc_probe();
258 if (found) {
259 mips_sc_enable();
260 mips_sc_prefetch_enable();
261 bcops = &mips_sc_ops;
262 }
263 return found;
264}
diff --git a/arch/mips/mm/sc-r5k.c b/arch/mips/mm/sc-r5k.c
new file mode 100644
index 000000000..736615d68
--- /dev/null
+++ b/arch/mips/mm/sc-r5k.c
@@ -0,0 +1,107 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 1997, 2001 Ralf Baechle (ralf@gnu.org),
4 * derived from r4xx0.c by David S. Miller (davem@davemloft.net).
5 */
6#include <linux/init.h>
7#include <linux/kernel.h>
8#include <linux/sched.h>
9#include <linux/mm.h>
10
11#include <asm/mipsregs.h>
12#include <asm/bcache.h>
13#include <asm/cacheops.h>
14#include <asm/page.h>
15#include <asm/mmu_context.h>
16#include <asm/r4kcache.h>
17
18/* Secondary cache size in bytes, if present. */
19static unsigned long scache_size;
20
21#define SC_LINE 32
22#define SC_PAGE (128*SC_LINE)
23
24static inline void blast_r5000_scache(void)
25{
26 unsigned long start = INDEX_BASE;
27 unsigned long end = start + scache_size;
28
29 while(start < end) {
30 cache_op(R5K_Page_Invalidate_S, start);
31 start += SC_PAGE;
32 }
33}
34
35static void r5k_dma_cache_inv_sc(unsigned long addr, unsigned long size)
36{
37 unsigned long end, a;
38
39 /* Catch bad driver code */
40 BUG_ON(size == 0);
41
42 if (size >= scache_size) {
43 blast_r5000_scache();
44 return;
45 }
46
47 /* On the R5000 secondary cache we cannot
48 * invalidate less than a page at a time.
49 * The secondary cache is physically indexed, write-through.
50 */
51 a = addr & ~(SC_PAGE - 1);
52 end = (addr + size - 1) & ~(SC_PAGE - 1);
53 while (a <= end) {
54 cache_op(R5K_Page_Invalidate_S, a);
55 a += SC_PAGE;
56 }
57}
58
59static void r5k_sc_enable(void)
60{
61 unsigned long flags;
62
63 local_irq_save(flags);
64 set_c0_config(R5K_CONF_SE);
65 blast_r5000_scache();
66 local_irq_restore(flags);
67}
68
69static void r5k_sc_disable(void)
70{
71 unsigned long flags;
72
73 local_irq_save(flags);
74 blast_r5000_scache();
75 clear_c0_config(R5K_CONF_SE);
76 local_irq_restore(flags);
77}
78
79static inline int __init r5k_sc_probe(void)
80{
81 unsigned long config = read_c0_config();
82
83 if (config & CONF_SC)
84 return 0;
85
86 scache_size = (512 * 1024) << ((config & R5K_CONF_SS) >> 20);
87
88 printk("R5000 SCACHE size %ldkB, linesize 32 bytes.\n",
89 scache_size >> 10);
90
91 return 1;
92}
93
94static struct bcache_ops r5k_sc_ops = {
95 .bc_enable = r5k_sc_enable,
96 .bc_disable = r5k_sc_disable,
97 .bc_wback_inv = r5k_dma_cache_inv_sc,
98 .bc_inv = r5k_dma_cache_inv_sc
99};
100
101void r5k_sc_init(void)
102{
103 if (r5k_sc_probe()) {
104 r5k_sc_enable();
105 bcops = &r5k_sc_ops;
106 }
107}
diff --git a/arch/mips/mm/sc-rm7k.c b/arch/mips/mm/sc-rm7k.c
new file mode 100644
index 000000000..e9e3777a7
--- /dev/null
+++ b/arch/mips/mm/sc-rm7k.c
@@ -0,0 +1,270 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * sc-rm7k.c: RM7000 cache management functions.
4 *
5 * Copyright (C) 1997, 2001, 2003, 2004 Ralf Baechle (ralf@linux-mips.org)
6 */
7
8#undef DEBUG
9
10#include <linux/kernel.h>
11#include <linux/mm.h>
12#include <linux/bitops.h>
13
14#include <asm/addrspace.h>
15#include <asm/bcache.h>
16#include <asm/cacheops.h>
17#include <asm/mipsregs.h>
18#include <asm/processor.h>
19#include <asm/sections.h>
20#include <asm/cacheflush.h> /* for run_uncached() */
21
22/* Primary cache parameters. */
23#define sc_lsize 32
24#define tc_pagesize (32*128)
25
26/* Secondary cache parameters. */
27#define scache_size (256*1024) /* Fixed to 256KiB on RM7000 */
28
29/* Tertiary cache parameters */
30#define tc_lsize 32
31
32extern unsigned long icache_way_size, dcache_way_size;
33static unsigned long tcache_size;
34
35#include <asm/r4kcache.h>
36
37static int rm7k_tcache_init;
38
39/*
40 * Writeback and invalidate the primary cache dcache before DMA.
41 * (XXX These need to be fixed ...)
42 */
43static void rm7k_sc_wback_inv(unsigned long addr, unsigned long size)
44{
45 unsigned long end, a;
46
47 pr_debug("rm7k_sc_wback_inv[%08lx,%08lx]", addr, size);
48
49 /* Catch bad driver code */
50 BUG_ON(size == 0);
51
52 blast_scache_range(addr, addr + size);
53
54 if (!rm7k_tcache_init)
55 return;
56
57 a = addr & ~(tc_pagesize - 1);
58 end = (addr + size - 1) & ~(tc_pagesize - 1);
59 while(1) {
60 invalidate_tcache_page(a); /* Page_Invalidate_T */
61 if (a == end)
62 break;
63 a += tc_pagesize;
64 }
65}
66
67static void rm7k_sc_inv(unsigned long addr, unsigned long size)
68{
69 unsigned long end, a;
70
71 pr_debug("rm7k_sc_inv[%08lx,%08lx]", addr, size);
72
73 /* Catch bad driver code */
74 BUG_ON(size == 0);
75
76 blast_inv_scache_range(addr, addr + size);
77
78 if (!rm7k_tcache_init)
79 return;
80
81 a = addr & ~(tc_pagesize - 1);
82 end = (addr + size - 1) & ~(tc_pagesize - 1);
83 while(1) {
84 invalidate_tcache_page(a); /* Page_Invalidate_T */
85 if (a == end)
86 break;
87 a += tc_pagesize;
88 }
89}
90
91static void blast_rm7k_tcache(void)
92{
93 unsigned long start = CKSEG0ADDR(0);
94 unsigned long end = start + tcache_size;
95
96 write_c0_taglo(0);
97
98 while (start < end) {
99 cache_op(Page_Invalidate_T, start);
100 start += tc_pagesize;
101 }
102}
103
104/*
105 * This function is executed in uncached address space.
106 */
107static void __rm7k_tc_enable(void)
108{
109 int i;
110
111 set_c0_config(RM7K_CONF_TE);
112
113 write_c0_taglo(0);
114 write_c0_taghi(0);
115
116 for (i = 0; i < tcache_size; i += tc_lsize)
117 cache_op(Index_Store_Tag_T, CKSEG0ADDR(i));
118}
119
120static void rm7k_tc_enable(void)
121{
122 if (read_c0_config() & RM7K_CONF_TE)
123 return;
124
125 BUG_ON(tcache_size == 0);
126
127 run_uncached(__rm7k_tc_enable);
128}
129
130/*
131 * This function is executed in uncached address space.
132 */
133static void __rm7k_sc_enable(void)
134{
135 int i;
136
137 set_c0_config(RM7K_CONF_SE);
138
139 write_c0_taglo(0);
140 write_c0_taghi(0);
141
142 for (i = 0; i < scache_size; i += sc_lsize)
143 cache_op(Index_Store_Tag_SD, CKSEG0ADDR(i));
144}
145
146static void rm7k_sc_enable(void)
147{
148 if (read_c0_config() & RM7K_CONF_SE)
149 return;
150
151 pr_info("Enabling secondary cache...\n");
152 run_uncached(__rm7k_sc_enable);
153
154 if (rm7k_tcache_init)
155 rm7k_tc_enable();
156}
157
158static void rm7k_tc_disable(void)
159{
160 unsigned long flags;
161
162 local_irq_save(flags);
163 blast_rm7k_tcache();
164 clear_c0_config(RM7K_CONF_TE);
165 local_irq_restore(flags);
166}
167
168static void rm7k_sc_disable(void)
169{
170 clear_c0_config(RM7K_CONF_SE);
171
172 if (rm7k_tcache_init)
173 rm7k_tc_disable();
174}
175
176static struct bcache_ops rm7k_sc_ops = {
177 .bc_enable = rm7k_sc_enable,
178 .bc_disable = rm7k_sc_disable,
179 .bc_wback_inv = rm7k_sc_wback_inv,
180 .bc_inv = rm7k_sc_inv
181};
182
183/*
184 * This is a probing function like the one found in c-r4k.c, we look for the
185 * wrap around point with different addresses.
186 */
187static void __probe_tcache(void)
188{
189 unsigned long flags, addr, begin, end, pow2;
190
191 begin = (unsigned long) &_stext;
192 begin &= ~((8 * 1024 * 1024) - 1);
193 end = begin + (8 * 1024 * 1024);
194
195 local_irq_save(flags);
196
197 set_c0_config(RM7K_CONF_TE);
198
199 /* Fill size-multiple lines with a valid tag */
200 pow2 = (256 * 1024);
201 for (addr = begin; addr <= end; addr = (begin + pow2)) {
202 unsigned long *p = (unsigned long *) addr;
203 __asm__ __volatile__("nop" : : "r" (*p));
204 pow2 <<= 1;
205 }
206
207 /* Load first line with a 0 tag, to check after */
208 write_c0_taglo(0);
209 write_c0_taghi(0);
210 cache_op(Index_Store_Tag_T, begin);
211
212 /* Look for the wrap-around */
213 pow2 = (512 * 1024);
214 for (addr = begin + (512 * 1024); addr <= end; addr = begin + pow2) {
215 cache_op(Index_Load_Tag_T, addr);
216 if (!read_c0_taglo())
217 break;
218 pow2 <<= 1;
219 }
220
221 addr -= begin;
222 tcache_size = addr;
223
224 clear_c0_config(RM7K_CONF_TE);
225
226 local_irq_restore(flags);
227}
228
229void rm7k_sc_init(void)
230{
231 struct cpuinfo_mips *c = &current_cpu_data;
232 unsigned int config = read_c0_config();
233
234 if ((config & RM7K_CONF_SC))
235 return;
236
237 c->scache.linesz = sc_lsize;
238 c->scache.ways = 4;
239 c->scache.waybit= __ffs(scache_size / c->scache.ways);
240 c->scache.waysize = scache_size / c->scache.ways;
241 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
242 printk(KERN_INFO "Secondary cache size %dK, linesize %d bytes.\n",
243 (scache_size >> 10), sc_lsize);
244
245 if (!(config & RM7K_CONF_SE))
246 rm7k_sc_enable();
247
248 bcops = &rm7k_sc_ops;
249
250 /*
251 * While we're at it let's deal with the tertiary cache.
252 */
253
254 rm7k_tcache_init = 0;
255 tcache_size = 0;
256
257 if (config & RM7K_CONF_TC)
258 return;
259
260 /*
261 * No efficient way to ask the hardware for the size of the tcache,
262 * so must probe for it.
263 */
264 run_uncached(__probe_tcache);
265 rm7k_tc_enable();
266 rm7k_tcache_init = 1;
267 c->tcache.linesz = tc_lsize;
268 c->tcache.ways = 1;
269 pr_info("Tertiary cache size %ldK.\n", (tcache_size >> 10));
270}
diff --git a/arch/mips/mm/tlb-funcs.S b/arch/mips/mm/tlb-funcs.S
new file mode 100644
index 000000000..00fef578c
--- /dev/null
+++ b/arch/mips/mm/tlb-funcs.S
@@ -0,0 +1,40 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Micro-assembler generated tlb handler functions.
7 *
8 * Copyright (C) 2013 Broadcom Corporation.
9 *
10 * Based on mm/page-funcs.c
11 * Copyright (C) 2012 MIPS Technologies, Inc.
12 * Copyright (C) 2012 Ralf Baechle <ralf@linux-mips.org>
13 */
14#include <asm/asm.h>
15#include <asm/export.h>
16#include <asm/regdef.h>
17
18#define FASTPATH_SIZE 128
19
20LEAF(tlbmiss_handler_setup_pgd)
211: j 1b /* Dummy, will be replaced. */
22 .space 64
23END(tlbmiss_handler_setup_pgd)
24EXPORT(tlbmiss_handler_setup_pgd_end)
25EXPORT_SYMBOL_GPL(tlbmiss_handler_setup_pgd)
26
27LEAF(handle_tlbm)
28 .space FASTPATH_SIZE * 4
29END(handle_tlbm)
30EXPORT(handle_tlbm_end)
31
32LEAF(handle_tlbs)
33 .space FASTPATH_SIZE * 4
34END(handle_tlbs)
35EXPORT(handle_tlbs_end)
36
37LEAF(handle_tlbl)
38 .space FASTPATH_SIZE * 4
39END(handle_tlbl)
40EXPORT(handle_tlbl_end)
diff --git a/arch/mips/mm/tlb-r3k.c b/arch/mips/mm/tlb-r3k.c
new file mode 100644
index 000000000..a36622ebe
--- /dev/null
+++ b/arch/mips/mm/tlb-r3k.c
@@ -0,0 +1,284 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * r2300.c: R2000 and R3000 specific mmu/cache code.
4 *
5 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
6 *
7 * with a lot of changes to make this thing work for R3000s
8 * Tx39XX R4k style caches added. HK
9 * Copyright (C) 1998, 1999, 2000 Harald Koerfgen
10 * Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
11 * Copyright (C) 2002 Ralf Baechle
12 * Copyright (C) 2002 Maciej W. Rozycki
13 */
14#include <linux/kernel.h>
15#include <linux/sched.h>
16#include <linux/smp.h>
17#include <linux/mm.h>
18
19#include <asm/page.h>
20#include <asm/mmu_context.h>
21#include <asm/tlbmisc.h>
22#include <asm/isadep.h>
23#include <asm/io.h>
24#include <asm/bootinfo.h>
25#include <asm/cpu.h>
26
27#undef DEBUG_TLB
28
29extern void build_tlb_refill_handler(void);
30
31/* CP0 hazard avoidance. */
32#define BARRIER \
33 __asm__ __volatile__( \
34 ".set push\n\t" \
35 ".set noreorder\n\t" \
36 "nop\n\t" \
37 ".set pop\n\t")
38
39int r3k_have_wired_reg; /* Should be in cpu_data? */
40
41/* TLB operations. */
42static void local_flush_tlb_from(int entry)
43{
44 unsigned long old_ctx;
45
46 old_ctx = read_c0_entryhi() & cpu_asid_mask(&current_cpu_data);
47 write_c0_entrylo0(0);
48 while (entry < current_cpu_data.tlbsize) {
49 write_c0_index(entry << 8);
50 write_c0_entryhi((entry | 0x80000) << 12);
51 entry++; /* BARRIER */
52 tlb_write_indexed();
53 }
54 write_c0_entryhi(old_ctx);
55}
56
57void local_flush_tlb_all(void)
58{
59 unsigned long flags;
60
61#ifdef DEBUG_TLB
62 printk("[tlball]");
63#endif
64 local_irq_save(flags);
65 local_flush_tlb_from(r3k_have_wired_reg ? read_c0_wired() : 8);
66 local_irq_restore(flags);
67}
68
69void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
70 unsigned long end)
71{
72 unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
73 struct mm_struct *mm = vma->vm_mm;
74 int cpu = smp_processor_id();
75
76 if (cpu_context(cpu, mm) != 0) {
77 unsigned long size, flags;
78
79#ifdef DEBUG_TLB
80 printk("[tlbrange<%lu,0x%08lx,0x%08lx>]",
81 cpu_context(cpu, mm) & asid_mask, start, end);
82#endif
83 local_irq_save(flags);
84 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
85 if (size <= current_cpu_data.tlbsize) {
86 int oldpid = read_c0_entryhi() & asid_mask;
87 int newpid = cpu_context(cpu, mm) & asid_mask;
88
89 start &= PAGE_MASK;
90 end += PAGE_SIZE - 1;
91 end &= PAGE_MASK;
92 while (start < end) {
93 int idx;
94
95 write_c0_entryhi(start | newpid);
96 start += PAGE_SIZE; /* BARRIER */
97 tlb_probe();
98 idx = read_c0_index();
99 write_c0_entrylo0(0);
100 write_c0_entryhi(KSEG0);
101 if (idx < 0) /* BARRIER */
102 continue;
103 tlb_write_indexed();
104 }
105 write_c0_entryhi(oldpid);
106 } else {
107 drop_mmu_context(mm);
108 }
109 local_irq_restore(flags);
110 }
111}
112
113void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
114{
115 unsigned long size, flags;
116
117#ifdef DEBUG_TLB
118 printk("[tlbrange<%lu,0x%08lx,0x%08lx>]", start, end);
119#endif
120 local_irq_save(flags);
121 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
122 if (size <= current_cpu_data.tlbsize) {
123 int pid = read_c0_entryhi();
124
125 start &= PAGE_MASK;
126 end += PAGE_SIZE - 1;
127 end &= PAGE_MASK;
128
129 while (start < end) {
130 int idx;
131
132 write_c0_entryhi(start);
133 start += PAGE_SIZE; /* BARRIER */
134 tlb_probe();
135 idx = read_c0_index();
136 write_c0_entrylo0(0);
137 write_c0_entryhi(KSEG0);
138 if (idx < 0) /* BARRIER */
139 continue;
140 tlb_write_indexed();
141 }
142 write_c0_entryhi(pid);
143 } else {
144 local_flush_tlb_all();
145 }
146 local_irq_restore(flags);
147}
148
149void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
150{
151 unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
152 int cpu = smp_processor_id();
153
154 if (cpu_context(cpu, vma->vm_mm) != 0) {
155 unsigned long flags;
156 int oldpid, newpid, idx;
157
158#ifdef DEBUG_TLB
159 printk("[tlbpage<%lu,0x%08lx>]", cpu_context(cpu, vma->vm_mm), page);
160#endif
161 newpid = cpu_context(cpu, vma->vm_mm) & asid_mask;
162 page &= PAGE_MASK;
163 local_irq_save(flags);
164 oldpid = read_c0_entryhi() & asid_mask;
165 write_c0_entryhi(page | newpid);
166 BARRIER;
167 tlb_probe();
168 idx = read_c0_index();
169 write_c0_entrylo0(0);
170 write_c0_entryhi(KSEG0);
171 if (idx < 0) /* BARRIER */
172 goto finish;
173 tlb_write_indexed();
174
175finish:
176 write_c0_entryhi(oldpid);
177 local_irq_restore(flags);
178 }
179}
180
181void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
182{
183 unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
184 unsigned long flags;
185 int idx, pid;
186
187 /*
188 * Handle debugger faulting in for debugee.
189 */
190 if (current->active_mm != vma->vm_mm)
191 return;
192
193 pid = read_c0_entryhi() & asid_mask;
194
195#ifdef DEBUG_TLB
196 if ((pid != (cpu_context(cpu, vma->vm_mm) & asid_mask)) || (cpu_context(cpu, vma->vm_mm) == 0)) {
197 printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%lu tlbpid=%d\n",
198 (cpu_context(cpu, vma->vm_mm)), pid);
199 }
200#endif
201
202 local_irq_save(flags);
203 address &= PAGE_MASK;
204 write_c0_entryhi(address | pid);
205 BARRIER;
206 tlb_probe();
207 idx = read_c0_index();
208 write_c0_entrylo0(pte_val(pte));
209 write_c0_entryhi(address | pid);
210 if (idx < 0) { /* BARRIER */
211 tlb_write_random();
212 } else {
213 tlb_write_indexed();
214 }
215 write_c0_entryhi(pid);
216 local_irq_restore(flags);
217}
218
219void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
220 unsigned long entryhi, unsigned long pagemask)
221{
222 unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
223 unsigned long flags;
224 unsigned long old_ctx;
225 static unsigned long wired = 0;
226
227 if (r3k_have_wired_reg) { /* TX39XX */
228 unsigned long old_pagemask;
229 unsigned long w;
230
231#ifdef DEBUG_TLB
232 printk("[tlbwired<entry lo0 %8x, hi %8x\n, pagemask %8x>]\n",
233 entrylo0, entryhi, pagemask);
234#endif
235
236 local_irq_save(flags);
237 /* Save old context and create impossible VPN2 value */
238 old_ctx = read_c0_entryhi() & asid_mask;
239 old_pagemask = read_c0_pagemask();
240 w = read_c0_wired();
241 write_c0_wired(w + 1);
242 write_c0_index(w << 8);
243 write_c0_pagemask(pagemask);
244 write_c0_entryhi(entryhi);
245 write_c0_entrylo0(entrylo0);
246 BARRIER;
247 tlb_write_indexed();
248
249 write_c0_entryhi(old_ctx);
250 write_c0_pagemask(old_pagemask);
251 local_flush_tlb_all();
252 local_irq_restore(flags);
253
254 } else if (wired < 8) {
255#ifdef DEBUG_TLB
256 printk("[tlbwired<entry lo0 %8x, hi %8x\n>]\n",
257 entrylo0, entryhi);
258#endif
259
260 local_irq_save(flags);
261 old_ctx = read_c0_entryhi() & asid_mask;
262 write_c0_entrylo0(entrylo0);
263 write_c0_entryhi(entryhi);
264 write_c0_index(wired);
265 wired++; /* BARRIER */
266 tlb_write_indexed();
267 write_c0_entryhi(old_ctx);
268 local_flush_tlb_all();
269 local_irq_restore(flags);
270 }
271}
272
273void tlb_init(void)
274{
275 switch (current_cpu_type()) {
276 case CPU_TX3922:
277 case CPU_TX3927:
278 r3k_have_wired_reg = 1;
279 write_c0_wired(0); /* Set to 8 on reset... */
280 break;
281 }
282 local_flush_tlb_from(0);
283 build_tlb_refill_handler();
284}
diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c
new file mode 100644
index 000000000..1b939abbe
--- /dev/null
+++ b/arch/mips/mm/tlb-r4k.c
@@ -0,0 +1,583 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
7 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
8 * Carsten Langgaard, carstenl@mips.com
9 * Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved.
10 */
11#include <linux/cpu_pm.h>
12#include <linux/init.h>
13#include <linux/sched.h>
14#include <linux/smp.h>
15#include <linux/mm.h>
16#include <linux/hugetlb.h>
17#include <linux/export.h>
18
19#include <asm/cpu.h>
20#include <asm/cpu-type.h>
21#include <asm/bootinfo.h>
22#include <asm/hazards.h>
23#include <asm/mmu_context.h>
24#include <asm/tlb.h>
25#include <asm/tlbmisc.h>
26
27extern void build_tlb_refill_handler(void);
28
29/*
30 * LOONGSON-2 has a 4 entry itlb which is a subset of jtlb, LOONGSON-3 has
31 * a 4 entry itlb and a 4 entry dtlb which are subsets of jtlb. Unfortunately,
32 * itlb/dtlb are not totally transparent to software.
33 */
34static inline void flush_micro_tlb(void)
35{
36 switch (current_cpu_type()) {
37 case CPU_LOONGSON2EF:
38 write_c0_diag(LOONGSON_DIAG_ITLB);
39 break;
40 case CPU_LOONGSON64:
41 write_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB);
42 break;
43 default:
44 break;
45 }
46}
47
48static inline void flush_micro_tlb_vm(struct vm_area_struct *vma)
49{
50 if (vma->vm_flags & VM_EXEC)
51 flush_micro_tlb();
52}
53
54void local_flush_tlb_all(void)
55{
56 unsigned long flags;
57 unsigned long old_ctx;
58 int entry, ftlbhighset;
59
60 local_irq_save(flags);
61 /* Save old context and create impossible VPN2 value */
62 old_ctx = read_c0_entryhi();
63 htw_stop();
64 write_c0_entrylo0(0);
65 write_c0_entrylo1(0);
66
67 entry = num_wired_entries();
68
69 /*
70 * Blast 'em all away.
71 * If there are any wired entries, fall back to iterating
72 */
73 if (cpu_has_tlbinv && !entry) {
74 if (current_cpu_data.tlbsizevtlb) {
75 write_c0_index(0);
76 mtc0_tlbw_hazard();
77 tlbinvf(); /* invalidate VTLB */
78 }
79 ftlbhighset = current_cpu_data.tlbsizevtlb +
80 current_cpu_data.tlbsizeftlbsets;
81 for (entry = current_cpu_data.tlbsizevtlb;
82 entry < ftlbhighset;
83 entry++) {
84 write_c0_index(entry);
85 mtc0_tlbw_hazard();
86 tlbinvf(); /* invalidate one FTLB set */
87 }
88 } else {
89 while (entry < current_cpu_data.tlbsize) {
90 /* Make sure all entries differ. */
91 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
92 write_c0_index(entry);
93 mtc0_tlbw_hazard();
94 tlb_write_indexed();
95 entry++;
96 }
97 }
98 tlbw_use_hazard();
99 write_c0_entryhi(old_ctx);
100 htw_start();
101 flush_micro_tlb();
102 local_irq_restore(flags);
103}
104EXPORT_SYMBOL(local_flush_tlb_all);
105
106void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
107 unsigned long end)
108{
109 struct mm_struct *mm = vma->vm_mm;
110 int cpu = smp_processor_id();
111
112 if (cpu_context(cpu, mm) != 0) {
113 unsigned long size, flags;
114
115 local_irq_save(flags);
116 start = round_down(start, PAGE_SIZE << 1);
117 end = round_up(end, PAGE_SIZE << 1);
118 size = (end - start) >> (PAGE_SHIFT + 1);
119 if (size <= (current_cpu_data.tlbsizeftlbsets ?
120 current_cpu_data.tlbsize / 8 :
121 current_cpu_data.tlbsize / 2)) {
122 unsigned long old_entryhi, old_mmid;
123 int newpid = cpu_asid(cpu, mm);
124
125 old_entryhi = read_c0_entryhi();
126 if (cpu_has_mmid) {
127 old_mmid = read_c0_memorymapid();
128 write_c0_memorymapid(newpid);
129 }
130
131 htw_stop();
132 while (start < end) {
133 int idx;
134
135 if (cpu_has_mmid)
136 write_c0_entryhi(start);
137 else
138 write_c0_entryhi(start | newpid);
139 start += (PAGE_SIZE << 1);
140 mtc0_tlbw_hazard();
141 tlb_probe();
142 tlb_probe_hazard();
143 idx = read_c0_index();
144 write_c0_entrylo0(0);
145 write_c0_entrylo1(0);
146 if (idx < 0)
147 continue;
148 /* Make sure all entries differ. */
149 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
150 mtc0_tlbw_hazard();
151 tlb_write_indexed();
152 }
153 tlbw_use_hazard();
154 write_c0_entryhi(old_entryhi);
155 if (cpu_has_mmid)
156 write_c0_memorymapid(old_mmid);
157 htw_start();
158 } else {
159 drop_mmu_context(mm);
160 }
161 flush_micro_tlb();
162 local_irq_restore(flags);
163 }
164}
165
166void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
167{
168 unsigned long size, flags;
169
170 local_irq_save(flags);
171 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
172 size = (size + 1) >> 1;
173 if (size <= (current_cpu_data.tlbsizeftlbsets ?
174 current_cpu_data.tlbsize / 8 :
175 current_cpu_data.tlbsize / 2)) {
176 int pid = read_c0_entryhi();
177
178 start &= (PAGE_MASK << 1);
179 end += ((PAGE_SIZE << 1) - 1);
180 end &= (PAGE_MASK << 1);
181 htw_stop();
182
183 while (start < end) {
184 int idx;
185
186 write_c0_entryhi(start);
187 start += (PAGE_SIZE << 1);
188 mtc0_tlbw_hazard();
189 tlb_probe();
190 tlb_probe_hazard();
191 idx = read_c0_index();
192 write_c0_entrylo0(0);
193 write_c0_entrylo1(0);
194 if (idx < 0)
195 continue;
196 /* Make sure all entries differ. */
197 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
198 mtc0_tlbw_hazard();
199 tlb_write_indexed();
200 }
201 tlbw_use_hazard();
202 write_c0_entryhi(pid);
203 htw_start();
204 } else {
205 local_flush_tlb_all();
206 }
207 flush_micro_tlb();
208 local_irq_restore(flags);
209}
210
211void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
212{
213 int cpu = smp_processor_id();
214
215 if (cpu_context(cpu, vma->vm_mm) != 0) {
216 unsigned long old_mmid;
217 unsigned long flags, old_entryhi;
218 int idx;
219
220 page &= (PAGE_MASK << 1);
221 local_irq_save(flags);
222 old_entryhi = read_c0_entryhi();
223 htw_stop();
224 if (cpu_has_mmid) {
225 old_mmid = read_c0_memorymapid();
226 write_c0_entryhi(page);
227 write_c0_memorymapid(cpu_asid(cpu, vma->vm_mm));
228 } else {
229 write_c0_entryhi(page | cpu_asid(cpu, vma->vm_mm));
230 }
231 mtc0_tlbw_hazard();
232 tlb_probe();
233 tlb_probe_hazard();
234 idx = read_c0_index();
235 write_c0_entrylo0(0);
236 write_c0_entrylo1(0);
237 if (idx < 0)
238 goto finish;
239 /* Make sure all entries differ. */
240 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
241 mtc0_tlbw_hazard();
242 tlb_write_indexed();
243 tlbw_use_hazard();
244
245 finish:
246 write_c0_entryhi(old_entryhi);
247 if (cpu_has_mmid)
248 write_c0_memorymapid(old_mmid);
249 htw_start();
250 flush_micro_tlb_vm(vma);
251 local_irq_restore(flags);
252 }
253}
254
255/*
256 * This one is only used for pages with the global bit set so we don't care
257 * much about the ASID.
258 */
259void local_flush_tlb_one(unsigned long page)
260{
261 unsigned long flags;
262 int oldpid, idx;
263
264 local_irq_save(flags);
265 oldpid = read_c0_entryhi();
266 htw_stop();
267 page &= (PAGE_MASK << 1);
268 write_c0_entryhi(page);
269 mtc0_tlbw_hazard();
270 tlb_probe();
271 tlb_probe_hazard();
272 idx = read_c0_index();
273 write_c0_entrylo0(0);
274 write_c0_entrylo1(0);
275 if (idx >= 0) {
276 /* Make sure all entries differ. */
277 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
278 mtc0_tlbw_hazard();
279 tlb_write_indexed();
280 tlbw_use_hazard();
281 }
282 write_c0_entryhi(oldpid);
283 htw_start();
284 flush_micro_tlb();
285 local_irq_restore(flags);
286}
287
288/*
289 * We will need multiple versions of update_mmu_cache(), one that just
290 * updates the TLB with the new pte(s), and another which also checks
291 * for the R4k "end of page" hardware bug and does the needy.
292 */
293void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
294{
295 unsigned long flags;
296 pgd_t *pgdp;
297 p4d_t *p4dp;
298 pud_t *pudp;
299 pmd_t *pmdp;
300 pte_t *ptep;
301 int idx, pid;
302
303 /*
304 * Handle debugger faulting in for debugee.
305 */
306 if (current->active_mm != vma->vm_mm)
307 return;
308
309 local_irq_save(flags);
310
311 htw_stop();
312 address &= (PAGE_MASK << 1);
313 if (cpu_has_mmid) {
314 write_c0_entryhi(address);
315 } else {
316 pid = read_c0_entryhi() & cpu_asid_mask(&current_cpu_data);
317 write_c0_entryhi(address | pid);
318 }
319 pgdp = pgd_offset(vma->vm_mm, address);
320 mtc0_tlbw_hazard();
321 tlb_probe();
322 tlb_probe_hazard();
323 p4dp = p4d_offset(pgdp, address);
324 pudp = pud_offset(p4dp, address);
325 pmdp = pmd_offset(pudp, address);
326 idx = read_c0_index();
327#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
328 /* this could be a huge page */
329 if (pmd_huge(*pmdp)) {
330 unsigned long lo;
331 write_c0_pagemask(PM_HUGE_MASK);
332 ptep = (pte_t *)pmdp;
333 lo = pte_to_entrylo(pte_val(*ptep));
334 write_c0_entrylo0(lo);
335 write_c0_entrylo1(lo + (HPAGE_SIZE >> 7));
336
337 mtc0_tlbw_hazard();
338 if (idx < 0)
339 tlb_write_random();
340 else
341 tlb_write_indexed();
342 tlbw_use_hazard();
343 write_c0_pagemask(PM_DEFAULT_MASK);
344 } else
345#endif
346 {
347 ptep = pte_offset_map(pmdp, address);
348
349#if defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32)
350#ifdef CONFIG_XPA
351 write_c0_entrylo0(pte_to_entrylo(ptep->pte_high));
352 if (cpu_has_xpa)
353 writex_c0_entrylo0(ptep->pte_low & _PFNX_MASK);
354 ptep++;
355 write_c0_entrylo1(pte_to_entrylo(ptep->pte_high));
356 if (cpu_has_xpa)
357 writex_c0_entrylo1(ptep->pte_low & _PFNX_MASK);
358#else
359 write_c0_entrylo0(ptep->pte_high);
360 ptep++;
361 write_c0_entrylo1(ptep->pte_high);
362#endif
363#else
364 write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++)));
365 write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep)));
366#endif
367 mtc0_tlbw_hazard();
368 if (idx < 0)
369 tlb_write_random();
370 else
371 tlb_write_indexed();
372 }
373 tlbw_use_hazard();
374 htw_start();
375 flush_micro_tlb_vm(vma);
376 local_irq_restore(flags);
377}
378
379void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
380 unsigned long entryhi, unsigned long pagemask)
381{
382#ifdef CONFIG_XPA
383 panic("Broken for XPA kernels");
384#else
385 unsigned int old_mmid;
386 unsigned long flags;
387 unsigned long wired;
388 unsigned long old_pagemask;
389 unsigned long old_ctx;
390
391 local_irq_save(flags);
392 if (cpu_has_mmid) {
393 old_mmid = read_c0_memorymapid();
394 write_c0_memorymapid(MMID_KERNEL_WIRED);
395 }
396 /* Save old context and create impossible VPN2 value */
397 old_ctx = read_c0_entryhi();
398 htw_stop();
399 old_pagemask = read_c0_pagemask();
400 wired = num_wired_entries();
401 write_c0_wired(wired + 1);
402 write_c0_index(wired);
403 tlbw_use_hazard(); /* What is the hazard here? */
404 write_c0_pagemask(pagemask);
405 write_c0_entryhi(entryhi);
406 write_c0_entrylo0(entrylo0);
407 write_c0_entrylo1(entrylo1);
408 mtc0_tlbw_hazard();
409 tlb_write_indexed();
410 tlbw_use_hazard();
411
412 write_c0_entryhi(old_ctx);
413 if (cpu_has_mmid)
414 write_c0_memorymapid(old_mmid);
415 tlbw_use_hazard(); /* What is the hazard here? */
416 htw_start();
417 write_c0_pagemask(old_pagemask);
418 local_flush_tlb_all();
419 local_irq_restore(flags);
420#endif
421}
422
423#ifdef CONFIG_TRANSPARENT_HUGEPAGE
424
425int has_transparent_hugepage(void)
426{
427 static unsigned int mask = -1;
428
429 if (mask == -1) { /* first call comes during __init */
430 unsigned long flags;
431
432 local_irq_save(flags);
433 write_c0_pagemask(PM_HUGE_MASK);
434 back_to_back_c0_hazard();
435 mask = read_c0_pagemask();
436 write_c0_pagemask(PM_DEFAULT_MASK);
437 local_irq_restore(flags);
438 }
439 return mask == PM_HUGE_MASK;
440}
441EXPORT_SYMBOL(has_transparent_hugepage);
442
443#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
444
445/*
446 * Used for loading TLB entries before trap_init() has started, when we
447 * don't actually want to add a wired entry which remains throughout the
448 * lifetime of the system
449 */
450
451int temp_tlb_entry;
452
453__init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1,
454 unsigned long entryhi, unsigned long pagemask)
455{
456 int ret = 0;
457 unsigned long flags;
458 unsigned long wired;
459 unsigned long old_pagemask;
460 unsigned long old_ctx;
461
462 local_irq_save(flags);
463 /* Save old context and create impossible VPN2 value */
464 htw_stop();
465 old_ctx = read_c0_entryhi();
466 old_pagemask = read_c0_pagemask();
467 wired = num_wired_entries();
468 if (--temp_tlb_entry < wired) {
469 printk(KERN_WARNING
470 "No TLB space left for add_temporary_entry\n");
471 ret = -ENOSPC;
472 goto out;
473 }
474
475 write_c0_index(temp_tlb_entry);
476 write_c0_pagemask(pagemask);
477 write_c0_entryhi(entryhi);
478 write_c0_entrylo0(entrylo0);
479 write_c0_entrylo1(entrylo1);
480 mtc0_tlbw_hazard();
481 tlb_write_indexed();
482 tlbw_use_hazard();
483
484 write_c0_entryhi(old_ctx);
485 write_c0_pagemask(old_pagemask);
486 htw_start();
487out:
488 local_irq_restore(flags);
489 return ret;
490}
491
492static int ntlb;
493static int __init set_ntlb(char *str)
494{
495 get_option(&str, &ntlb);
496 return 1;
497}
498
499__setup("ntlb=", set_ntlb);
500
501/*
502 * Configure TLB (for init or after a CPU has been powered off).
503 */
504static void r4k_tlb_configure(void)
505{
506 /*
507 * You should never change this register:
508 * - On R4600 1.7 the tlbp never hits for pages smaller than
509 * the value in the c0_pagemask register.
510 * - The entire mm handling assumes the c0_pagemask register to
511 * be set to fixed-size pages.
512 */
513 write_c0_pagemask(PM_DEFAULT_MASK);
514 back_to_back_c0_hazard();
515 if (read_c0_pagemask() != PM_DEFAULT_MASK)
516 panic("MMU doesn't support PAGE_SIZE=0x%lx", PAGE_SIZE);
517
518 write_c0_wired(0);
519 if (current_cpu_type() == CPU_R10000 ||
520 current_cpu_type() == CPU_R12000 ||
521 current_cpu_type() == CPU_R14000 ||
522 current_cpu_type() == CPU_R16000)
523 write_c0_framemask(0);
524
525 if (cpu_has_rixi) {
526 /*
527 * Enable the no read, no exec bits, and enable large physical
528 * address.
529 */
530#ifdef CONFIG_64BIT
531 set_c0_pagegrain(PG_RIE | PG_XIE | PG_ELPA);
532#else
533 set_c0_pagegrain(PG_RIE | PG_XIE);
534#endif
535 }
536
537 temp_tlb_entry = current_cpu_data.tlbsize - 1;
538
539 /* From this point on the ARC firmware is dead. */
540 local_flush_tlb_all();
541
542 /* Did I tell you that ARC SUCKS? */
543}
544
545void tlb_init(void)
546{
547 r4k_tlb_configure();
548
549 if (ntlb) {
550 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) {
551 int wired = current_cpu_data.tlbsize - ntlb;
552 write_c0_wired(wired);
553 write_c0_index(wired-1);
554 printk("Restricting TLB to %d entries\n", ntlb);
555 } else
556 printk("Ignoring invalid argument ntlb=%d\n", ntlb);
557 }
558
559 build_tlb_refill_handler();
560}
561
562static int r4k_tlb_pm_notifier(struct notifier_block *self, unsigned long cmd,
563 void *v)
564{
565 switch (cmd) {
566 case CPU_PM_ENTER_FAILED:
567 case CPU_PM_EXIT:
568 r4k_tlb_configure();
569 break;
570 }
571
572 return NOTIFY_OK;
573}
574
575static struct notifier_block r4k_tlb_pm_notifier_block = {
576 .notifier_call = r4k_tlb_pm_notifier,
577};
578
579static int __init r4k_tlb_init_pm(void)
580{
581 return cpu_pm_register_notifier(&r4k_tlb_pm_notifier_block);
582}
583arch_initcall(r4k_tlb_init_pm);
diff --git a/arch/mips/mm/tlbex-fault.S b/arch/mips/mm/tlbex-fault.S
new file mode 100644
index 000000000..77db401fc
--- /dev/null
+++ b/arch/mips/mm/tlbex-fault.S
@@ -0,0 +1,28 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1999 Ralf Baechle
7 * Copyright (C) 1999 Silicon Graphics, Inc.
8 */
9#include <asm/mipsregs.h>
10#include <asm/regdef.h>
11#include <asm/stackframe.h>
12
13 .macro tlb_do_page_fault, write
14 NESTED(tlb_do_page_fault_\write, PT_SIZE, sp)
15 .cfi_signal_frame
16 SAVE_ALL docfi=1
17 MFC0 a2, CP0_BADVADDR
18 KMODE
19 move a0, sp
20 REG_S a2, PT_BVADDR(sp)
21 li a1, \write
22 jal do_page_fault
23 j ret_from_exception
24 END(tlb_do_page_fault_\write)
25 .endm
26
27 tlb_do_page_fault 0
28 tlb_do_page_fault 1
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
new file mode 100644
index 000000000..e8e3635dd
--- /dev/null
+++ b/arch/mips/mm/tlbex.c
@@ -0,0 +1,2661 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Synthesize TLB refill handlers at runtime.
7 *
8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
9 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki
10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
11 * Copyright (C) 2008, 2009 Cavium Networks, Inc.
12 * Copyright (C) 2011 MIPS Technologies, Inc.
13 *
14 * ... and the days got worse and worse and now you see
15 * I've gone completely out of my mind.
16 *
17 * They're coming to take me a away haha
18 * they're coming to take me a away hoho hihi haha
19 * to the funny farm where code is beautiful all the time ...
20 *
21 * (Condolences to Napoleon XIV)
22 */
23
24#include <linux/bug.h>
25#include <linux/export.h>
26#include <linux/kernel.h>
27#include <linux/types.h>
28#include <linux/smp.h>
29#include <linux/string.h>
30#include <linux/cache.h>
31#include <linux/pgtable.h>
32
33#include <asm/cacheflush.h>
34#include <asm/cpu-type.h>
35#include <asm/mmu_context.h>
36#include <asm/war.h>
37#include <asm/uasm.h>
38#include <asm/setup.h>
39#include <asm/tlbex.h>
40
41static int mips_xpa_disabled;
42
43static int __init xpa_disable(char *s)
44{
45 mips_xpa_disabled = 1;
46
47 return 1;
48}
49
50__setup("noxpa", xpa_disable);
51
52/*
53 * TLB load/store/modify handlers.
54 *
55 * Only the fastpath gets synthesized at runtime, the slowpath for
56 * do_page_fault remains normal asm.
57 */
58extern void tlb_do_page_fault_0(void);
59extern void tlb_do_page_fault_1(void);
60
61struct work_registers {
62 int r1;
63 int r2;
64 int r3;
65};
66
67struct tlb_reg_save {
68 unsigned long a;
69 unsigned long b;
70} ____cacheline_aligned_in_smp;
71
72static struct tlb_reg_save handler_reg_save[NR_CPUS];
73
74static inline int r45k_bvahwbug(void)
75{
76 /* XXX: We should probe for the presence of this bug, but we don't. */
77 return 0;
78}
79
80static inline int r4k_250MHZhwbug(void)
81{
82 /* XXX: We should probe for the presence of this bug, but we don't. */
83 return 0;
84}
85
86extern int sb1250_m3_workaround_needed(void);
87
88static inline int __maybe_unused bcm1250_m3_war(void)
89{
90 if (IS_ENABLED(CONFIG_SB1_PASS_2_WORKAROUNDS))
91 return sb1250_m3_workaround_needed();
92 return 0;
93}
94
95static inline int __maybe_unused r10000_llsc_war(void)
96{
97 return IS_ENABLED(CONFIG_WAR_R10000_LLSC);
98}
99
100static int use_bbit_insns(void)
101{
102 switch (current_cpu_type()) {
103 case CPU_CAVIUM_OCTEON:
104 case CPU_CAVIUM_OCTEON_PLUS:
105 case CPU_CAVIUM_OCTEON2:
106 case CPU_CAVIUM_OCTEON3:
107 return 1;
108 default:
109 return 0;
110 }
111}
112
113static int use_lwx_insns(void)
114{
115 switch (current_cpu_type()) {
116 case CPU_CAVIUM_OCTEON2:
117 case CPU_CAVIUM_OCTEON3:
118 return 1;
119 default:
120 return 0;
121 }
122}
123#if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \
124 CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0
125static bool scratchpad_available(void)
126{
127 return true;
128}
129static int scratchpad_offset(int i)
130{
131 /*
132 * CVMSEG starts at address -32768 and extends for
133 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines.
134 */
135 i += 1; /* Kernel use starts at the top and works down. */
136 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768;
137}
138#else
139static bool scratchpad_available(void)
140{
141 return false;
142}
143static int scratchpad_offset(int i)
144{
145 BUG();
146 /* Really unreachable, but evidently some GCC want this. */
147 return 0;
148}
149#endif
150/*
151 * Found by experiment: At least some revisions of the 4kc throw under
152 * some circumstances a machine check exception, triggered by invalid
153 * values in the index register. Delaying the tlbp instruction until
154 * after the next branch, plus adding an additional nop in front of
155 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows
156 * why; it's not an issue caused by the core RTL.
157 *
158 */
159static int m4kc_tlbp_war(void)
160{
161 return current_cpu_type() == CPU_4KC;
162}
163
164/* Handle labels (which must be positive integers). */
165enum label_id {
166 label_second_part = 1,
167 label_leave,
168 label_vmalloc,
169 label_vmalloc_done,
170 label_tlbw_hazard_0,
171 label_split = label_tlbw_hazard_0 + 8,
172 label_tlbl_goaround1,
173 label_tlbl_goaround2,
174 label_nopage_tlbl,
175 label_nopage_tlbs,
176 label_nopage_tlbm,
177 label_smp_pgtable_change,
178 label_r3000_write_probe_fail,
179 label_large_segbits_fault,
180#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
181 label_tlb_huge_update,
182#endif
183};
184
185UASM_L_LA(_second_part)
186UASM_L_LA(_leave)
187UASM_L_LA(_vmalloc)
188UASM_L_LA(_vmalloc_done)
189/* _tlbw_hazard_x is handled differently. */
190UASM_L_LA(_split)
191UASM_L_LA(_tlbl_goaround1)
192UASM_L_LA(_tlbl_goaround2)
193UASM_L_LA(_nopage_tlbl)
194UASM_L_LA(_nopage_tlbs)
195UASM_L_LA(_nopage_tlbm)
196UASM_L_LA(_smp_pgtable_change)
197UASM_L_LA(_r3000_write_probe_fail)
198UASM_L_LA(_large_segbits_fault)
199#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
200UASM_L_LA(_tlb_huge_update)
201#endif
202
203static int hazard_instance;
204
205static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance)
206{
207 switch (instance) {
208 case 0 ... 7:
209 uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance);
210 return;
211 default:
212 BUG();
213 }
214}
215
216static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance)
217{
218 switch (instance) {
219 case 0 ... 7:
220 uasm_build_label(l, *p, label_tlbw_hazard_0 + instance);
221 break;
222 default:
223 BUG();
224 }
225}
226
227/*
228 * pgtable bits are assigned dynamically depending on processor feature
229 * and statically based on kernel configuration. This spits out the actual
230 * values the kernel is using. Required to make sense from disassembled
231 * TLB exception handlers.
232 */
233static void output_pgtable_bits_defines(void)
234{
235#define pr_define(fmt, ...) \
236 pr_debug("#define " fmt, ##__VA_ARGS__)
237
238 pr_debug("#include <asm/asm.h>\n");
239 pr_debug("#include <asm/regdef.h>\n");
240 pr_debug("\n");
241
242 pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT);
243 pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT);
244 pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT);
245 pr_define("_PAGE_ACCESSED_SHIFT %d\n", _PAGE_ACCESSED_SHIFT);
246 pr_define("_PAGE_MODIFIED_SHIFT %d\n", _PAGE_MODIFIED_SHIFT);
247#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
248 pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT);
249#endif
250#ifdef _PAGE_NO_EXEC_SHIFT
251 if (cpu_has_rixi)
252 pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT);
253#endif
254 pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT);
255 pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT);
256 pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT);
257 pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT);
258 pr_debug("\n");
259}
260
261static inline void dump_handler(const char *symbol, const void *start, const void *end)
262{
263 unsigned int count = (end - start) / sizeof(u32);
264 const u32 *handler = start;
265 int i;
266
267 pr_debug("LEAF(%s)\n", symbol);
268
269 pr_debug("\t.set push\n");
270 pr_debug("\t.set noreorder\n");
271
272 for (i = 0; i < count; i++)
273 pr_debug("\t.word\t0x%08x\t\t# %p\n", handler[i], &handler[i]);
274
275 pr_debug("\t.set\tpop\n");
276
277 pr_debug("\tEND(%s)\n", symbol);
278}
279
280/* The only general purpose registers allowed in TLB handlers. */
281#define K0 26
282#define K1 27
283
284/* Some CP0 registers */
285#define C0_INDEX 0, 0
286#define C0_ENTRYLO0 2, 0
287#define C0_TCBIND 2, 2
288#define C0_ENTRYLO1 3, 0
289#define C0_CONTEXT 4, 0
290#define C0_PAGEMASK 5, 0
291#define C0_PWBASE 5, 5
292#define C0_PWFIELD 5, 6
293#define C0_PWSIZE 5, 7
294#define C0_PWCTL 6, 6
295#define C0_BADVADDR 8, 0
296#define C0_PGD 9, 7
297#define C0_ENTRYHI 10, 0
298#define C0_EPC 14, 0
299#define C0_XCONTEXT 20, 0
300
301#ifdef CONFIG_64BIT
302# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT)
303#else
304# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT)
305#endif
306
307/* The worst case length of the handler is around 18 instructions for
308 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
309 * Maximum space available is 32 instructions for R3000 and 64
310 * instructions for R4000.
311 *
312 * We deliberately chose a buffer size of 128, so we won't scribble
313 * over anything important on overflow before we panic.
314 */
315static u32 tlb_handler[128];
316
317/* simply assume worst case size for labels and relocs */
318static struct uasm_label labels[128];
319static struct uasm_reloc relocs[128];
320
321static int check_for_high_segbits;
322static bool fill_includes_sw_bits;
323
324static unsigned int kscratch_used_mask;
325
326static inline int __maybe_unused c0_kscratch(void)
327{
328 switch (current_cpu_type()) {
329 case CPU_XLP:
330 case CPU_XLR:
331 return 22;
332 default:
333 return 31;
334 }
335}
336
337static int allocate_kscratch(void)
338{
339 int r;
340 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask;
341
342 r = ffs(a);
343
344 if (r == 0)
345 return -1;
346
347 r--; /* make it zero based */
348
349 kscratch_used_mask |= (1 << r);
350
351 return r;
352}
353
354static int scratch_reg;
355int pgd_reg;
356EXPORT_SYMBOL_GPL(pgd_reg);
357enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch};
358
359static struct work_registers build_get_work_registers(u32 **p)
360{
361 struct work_registers r;
362
363 if (scratch_reg >= 0) {
364 /* Save in CPU local C0_KScratch? */
365 UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg);
366 r.r1 = K0;
367 r.r2 = K1;
368 r.r3 = 1;
369 return r;
370 }
371
372 if (num_possible_cpus() > 1) {
373 /* Get smp_processor_id */
374 UASM_i_CPUID_MFC0(p, K0, SMP_CPUID_REG);
375 UASM_i_SRL_SAFE(p, K0, K0, SMP_CPUID_REGSHIFT);
376
377 /* handler_reg_save index in K0 */
378 UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save)));
379
380 UASM_i_LA(p, K1, (long)&handler_reg_save);
381 UASM_i_ADDU(p, K0, K0, K1);
382 } else {
383 UASM_i_LA(p, K0, (long)&handler_reg_save);
384 }
385 /* K0 now points to save area, save $1 and $2 */
386 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0);
387 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0);
388
389 r.r1 = K1;
390 r.r2 = 1;
391 r.r3 = 2;
392 return r;
393}
394
395static void build_restore_work_registers(u32 **p)
396{
397 if (scratch_reg >= 0) {
398 uasm_i_ehb(p);
399 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
400 return;
401 }
402 /* K0 already points to save area, restore $1 and $2 */
403 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0);
404 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0);
405}
406
407#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
408
409/*
410 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
411 * we cannot do r3000 under these circumstances.
412 *
413 * The R3000 TLB handler is simple.
414 */
415static void build_r3000_tlb_refill_handler(void)
416{
417 long pgdc = (long)pgd_current;
418 u32 *p;
419
420 memset(tlb_handler, 0, sizeof(tlb_handler));
421 p = tlb_handler;
422
423 uasm_i_mfc0(&p, K0, C0_BADVADDR);
424 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */
425 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1);
426 uasm_i_srl(&p, K0, K0, 22); /* load delay */
427 uasm_i_sll(&p, K0, K0, 2);
428 uasm_i_addu(&p, K1, K1, K0);
429 uasm_i_mfc0(&p, K0, C0_CONTEXT);
430 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */
431 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */
432 uasm_i_addu(&p, K1, K1, K0);
433 uasm_i_lw(&p, K0, 0, K1);
434 uasm_i_nop(&p); /* load delay */
435 uasm_i_mtc0(&p, K0, C0_ENTRYLO0);
436 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
437 uasm_i_tlbwr(&p); /* cp0 delay */
438 uasm_i_jr(&p, K1);
439 uasm_i_rfe(&p); /* branch delay */
440
441 if (p > tlb_handler + 32)
442 panic("TLB refill handler space exceeded");
443
444 pr_debug("Wrote TLB refill handler (%u instructions).\n",
445 (unsigned int)(p - tlb_handler));
446
447 memcpy((void *)ebase, tlb_handler, 0x80);
448 local_flush_icache_range(ebase, ebase + 0x80);
449 dump_handler("r3000_tlb_refill", (u32 *)ebase, (u32 *)(ebase + 0x80));
450}
451#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
452
453/*
454 * The R4000 TLB handler is much more complicated. We have two
455 * consecutive handler areas with 32 instructions space each.
456 * Since they aren't used at the same time, we can overflow in the
457 * other one.To keep things simple, we first assume linear space,
458 * then we relocate it to the final handler layout as needed.
459 */
460static u32 final_handler[64];
461
462/*
463 * Hazards
464 *
465 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0:
466 * 2. A timing hazard exists for the TLBP instruction.
467 *
468 * stalling_instruction
469 * TLBP
470 *
471 * The JTLB is being read for the TLBP throughout the stall generated by the
472 * previous instruction. This is not really correct as the stalling instruction
473 * can modify the address used to access the JTLB. The failure symptom is that
474 * the TLBP instruction will use an address created for the stalling instruction
475 * and not the address held in C0_ENHI and thus report the wrong results.
476 *
477 * The software work-around is to not allow the instruction preceding the TLBP
478 * to stall - make it an NOP or some other instruction guaranteed not to stall.
479 *
480 * Errata 2 will not be fixed. This errata is also on the R5000.
481 *
482 * As if we MIPS hackers wouldn't know how to nop pipelines happy ...
483 */
484static void __maybe_unused build_tlb_probe_entry(u32 **p)
485{
486 switch (current_cpu_type()) {
487 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */
488 case CPU_R4600:
489 case CPU_R4700:
490 case CPU_R5000:
491 case CPU_NEVADA:
492 uasm_i_nop(p);
493 uasm_i_tlbp(p);
494 break;
495
496 default:
497 uasm_i_tlbp(p);
498 break;
499 }
500}
501
502void build_tlb_write_entry(u32 **p, struct uasm_label **l,
503 struct uasm_reloc **r,
504 enum tlb_write_entry wmode)
505{
506 void(*tlbw)(u32 **) = NULL;
507
508 switch (wmode) {
509 case tlb_random: tlbw = uasm_i_tlbwr; break;
510 case tlb_indexed: tlbw = uasm_i_tlbwi; break;
511 }
512
513 if (cpu_has_mips_r2_r6) {
514 if (cpu_has_mips_r2_exec_hazard)
515 uasm_i_ehb(p);
516 tlbw(p);
517 return;
518 }
519
520 switch (current_cpu_type()) {
521 case CPU_R4000PC:
522 case CPU_R4000SC:
523 case CPU_R4000MC:
524 case CPU_R4400PC:
525 case CPU_R4400SC:
526 case CPU_R4400MC:
527 /*
528 * This branch uses up a mtc0 hazard nop slot and saves
529 * two nops after the tlbw instruction.
530 */
531 uasm_bgezl_hazard(p, r, hazard_instance);
532 tlbw(p);
533 uasm_bgezl_label(l, p, hazard_instance);
534 hazard_instance++;
535 uasm_i_nop(p);
536 break;
537
538 case CPU_R4600:
539 case CPU_R4700:
540 uasm_i_nop(p);
541 tlbw(p);
542 uasm_i_nop(p);
543 break;
544
545 case CPU_R5000:
546 case CPU_NEVADA:
547 uasm_i_nop(p); /* QED specifies 2 nops hazard */
548 uasm_i_nop(p); /* QED specifies 2 nops hazard */
549 tlbw(p);
550 break;
551
552 case CPU_5KC:
553 case CPU_TX49XX:
554 case CPU_PR4450:
555 case CPU_XLR:
556 uasm_i_nop(p);
557 tlbw(p);
558 break;
559
560 case CPU_R10000:
561 case CPU_R12000:
562 case CPU_R14000:
563 case CPU_R16000:
564 case CPU_4KC:
565 case CPU_4KEC:
566 case CPU_M14KC:
567 case CPU_M14KEC:
568 case CPU_SB1:
569 case CPU_SB1A:
570 case CPU_4KSC:
571 case CPU_20KC:
572 case CPU_25KF:
573 case CPU_BMIPS32:
574 case CPU_BMIPS3300:
575 case CPU_BMIPS4350:
576 case CPU_BMIPS4380:
577 case CPU_BMIPS5000:
578 case CPU_LOONGSON2EF:
579 case CPU_LOONGSON64:
580 case CPU_R5500:
581 if (m4kc_tlbp_war())
582 uasm_i_nop(p);
583 fallthrough;
584 case CPU_ALCHEMY:
585 tlbw(p);
586 break;
587
588 case CPU_RM7000:
589 uasm_i_nop(p);
590 uasm_i_nop(p);
591 uasm_i_nop(p);
592 uasm_i_nop(p);
593 tlbw(p);
594 break;
595
596 case CPU_VR4111:
597 case CPU_VR4121:
598 case CPU_VR4122:
599 case CPU_VR4181:
600 case CPU_VR4181A:
601 uasm_i_nop(p);
602 uasm_i_nop(p);
603 tlbw(p);
604 uasm_i_nop(p);
605 uasm_i_nop(p);
606 break;
607
608 case CPU_VR4131:
609 case CPU_VR4133:
610 uasm_i_nop(p);
611 uasm_i_nop(p);
612 tlbw(p);
613 break;
614
615 case CPU_XBURST:
616 tlbw(p);
617 uasm_i_nop(p);
618 break;
619
620 default:
621 panic("No TLB refill handler yet (CPU type: %d)",
622 current_cpu_type());
623 break;
624 }
625}
626EXPORT_SYMBOL_GPL(build_tlb_write_entry);
627
628static __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
629 unsigned int reg)
630{
631 if (_PAGE_GLOBAL_SHIFT == 0) {
632 /* pte_t is already in EntryLo format */
633 return;
634 }
635
636 if (cpu_has_rixi && _PAGE_NO_EXEC != 0) {
637 if (fill_includes_sw_bits) {
638 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL));
639 } else {
640 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC));
641 UASM_i_ROTR(p, reg, reg,
642 ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
643 }
644 } else {
645#ifdef CONFIG_PHYS_ADDR_T_64BIT
646 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL));
647#else
648 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL));
649#endif
650 }
651}
652
653#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
654
655static void build_restore_pagemask(u32 **p, struct uasm_reloc **r,
656 unsigned int tmp, enum label_id lid,
657 int restore_scratch)
658{
659 if (restore_scratch) {
660 /*
661 * Ensure the MFC0 below observes the value written to the
662 * KScratch register by the prior MTC0.
663 */
664 if (scratch_reg >= 0)
665 uasm_i_ehb(p);
666
667 /* Reset default page size */
668 if (PM_DEFAULT_MASK >> 16) {
669 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
670 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
671 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
672 uasm_il_b(p, r, lid);
673 } else if (PM_DEFAULT_MASK) {
674 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
675 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
676 uasm_il_b(p, r, lid);
677 } else {
678 uasm_i_mtc0(p, 0, C0_PAGEMASK);
679 uasm_il_b(p, r, lid);
680 }
681 if (scratch_reg >= 0)
682 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
683 else
684 UASM_i_LW(p, 1, scratchpad_offset(0), 0);
685 } else {
686 /* Reset default page size */
687 if (PM_DEFAULT_MASK >> 16) {
688 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
689 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
690 uasm_il_b(p, r, lid);
691 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
692 } else if (PM_DEFAULT_MASK) {
693 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
694 uasm_il_b(p, r, lid);
695 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
696 } else {
697 uasm_il_b(p, r, lid);
698 uasm_i_mtc0(p, 0, C0_PAGEMASK);
699 }
700 }
701}
702
703static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l,
704 struct uasm_reloc **r,
705 unsigned int tmp,
706 enum tlb_write_entry wmode,
707 int restore_scratch)
708{
709 /* Set huge page tlb entry size */
710 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
711 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff);
712 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
713
714 build_tlb_write_entry(p, l, r, wmode);
715
716 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch);
717}
718
719/*
720 * Check if Huge PTE is present, if so then jump to LABEL.
721 */
722static void
723build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
724 unsigned int pmd, int lid)
725{
726 UASM_i_LW(p, tmp, 0, pmd);
727 if (use_bbit_insns()) {
728 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid);
729 } else {
730 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE);
731 uasm_il_bnez(p, r, tmp, lid);
732 }
733}
734
735static void build_huge_update_entries(u32 **p, unsigned int pte,
736 unsigned int tmp)
737{
738 int small_sequence;
739
740 /*
741 * A huge PTE describes an area the size of the
742 * configured huge page size. This is twice the
743 * of the large TLB entry size we intend to use.
744 * A TLB entry half the size of the configured
745 * huge page size is configured into entrylo0
746 * and entrylo1 to cover the contiguous huge PTE
747 * address space.
748 */
749 small_sequence = (HPAGE_SIZE >> 7) < 0x10000;
750
751 /* We can clobber tmp. It isn't used after this.*/
752 if (!small_sequence)
753 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16));
754
755 build_convert_pte_to_entrylo(p, pte);
756 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */
757 /* convert to entrylo1 */
758 if (small_sequence)
759 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7);
760 else
761 UASM_i_ADDU(p, pte, pte, tmp);
762
763 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */
764}
765
766static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r,
767 struct uasm_label **l,
768 unsigned int pte,
769 unsigned int ptr,
770 unsigned int flush)
771{
772#ifdef CONFIG_SMP
773 UASM_i_SC(p, pte, 0, ptr);
774 uasm_il_beqz(p, r, pte, label_tlb_huge_update);
775 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */
776#else
777 UASM_i_SW(p, pte, 0, ptr);
778#endif
779 if (cpu_has_ftlb && flush) {
780 BUG_ON(!cpu_has_tlbinv);
781
782 UASM_i_MFC0(p, ptr, C0_ENTRYHI);
783 uasm_i_ori(p, ptr, ptr, MIPS_ENTRYHI_EHINV);
784 UASM_i_MTC0(p, ptr, C0_ENTRYHI);
785 build_tlb_write_entry(p, l, r, tlb_indexed);
786
787 uasm_i_xori(p, ptr, ptr, MIPS_ENTRYHI_EHINV);
788 UASM_i_MTC0(p, ptr, C0_ENTRYHI);
789 build_huge_update_entries(p, pte, ptr);
790 build_huge_tlb_write_entry(p, l, r, pte, tlb_random, 0);
791
792 return;
793 }
794
795 build_huge_update_entries(p, pte, ptr);
796 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0);
797}
798#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */
799
800#ifdef CONFIG_64BIT
801/*
802 * TMP and PTR are scratch.
803 * TMP will be clobbered, PTR will hold the pmd entry.
804 */
805void build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
806 unsigned int tmp, unsigned int ptr)
807{
808#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
809 long pgdc = (long)pgd_current;
810#endif
811 /*
812 * The vmalloc handling is not in the hotpath.
813 */
814 uasm_i_dmfc0(p, tmp, C0_BADVADDR);
815
816 if (check_for_high_segbits) {
817 /*
818 * The kernel currently implicitely assumes that the
819 * MIPS SEGBITS parameter for the processor is
820 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never
821 * allocate virtual addresses outside the maximum
822 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But
823 * that doesn't prevent user code from accessing the
824 * higher xuseg addresses. Here, we make sure that
825 * everything but the lower xuseg addresses goes down
826 * the module_alloc/vmalloc path.
827 */
828 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
829 uasm_il_bnez(p, r, ptr, label_vmalloc);
830 } else {
831 uasm_il_bltz(p, r, tmp, label_vmalloc);
832 }
833 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
834
835 if (pgd_reg != -1) {
836 /* pgd is in pgd_reg */
837 if (cpu_has_ldpte)
838 UASM_i_MFC0(p, ptr, C0_PWBASE);
839 else
840 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
841 } else {
842#if defined(CONFIG_MIPS_PGD_C0_CONTEXT)
843 /*
844 * &pgd << 11 stored in CONTEXT [23..63].
845 */
846 UASM_i_MFC0(p, ptr, C0_CONTEXT);
847
848 /* Clear lower 23 bits of context. */
849 uasm_i_dins(p, ptr, 0, 0, 23);
850
851 /* 1 0 1 0 1 << 6 xkphys cached */
852 uasm_i_ori(p, ptr, ptr, 0x540);
853 uasm_i_drotr(p, ptr, ptr, 11);
854#elif defined(CONFIG_SMP)
855 UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG);
856 uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT);
857 UASM_i_LA_mostly(p, tmp, pgdc);
858 uasm_i_daddu(p, ptr, ptr, tmp);
859 uasm_i_dmfc0(p, tmp, C0_BADVADDR);
860 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
861#else
862 UASM_i_LA_mostly(p, ptr, pgdc);
863 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
864#endif
865 }
866
867 uasm_l_vmalloc_done(l, *p);
868
869 /* get pgd offset in bytes */
870 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3);
871
872 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
873 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
874#ifndef __PAGETABLE_PUD_FOLDED
875 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
876 uasm_i_ld(p, ptr, 0, ptr); /* get pud pointer */
877 uasm_i_dsrl_safe(p, tmp, tmp, PUD_SHIFT - 3); /* get pud offset in bytes */
878 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PUD - 1) << 3);
879 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pud offset */
880#endif
881#ifndef __PAGETABLE_PMD_FOLDED
882 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
883 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */
884 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
885 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
886 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
887#endif
888}
889EXPORT_SYMBOL_GPL(build_get_pmde64);
890
891/*
892 * BVADDR is the faulting address, PTR is scratch.
893 * PTR will hold the pgd for vmalloc.
894 */
895static void
896build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
897 unsigned int bvaddr, unsigned int ptr,
898 enum vmalloc64_mode mode)
899{
900 long swpd = (long)swapper_pg_dir;
901 int single_insn_swpd;
902 int did_vmalloc_branch = 0;
903
904 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd);
905
906 uasm_l_vmalloc(l, *p);
907
908 if (mode != not_refill && check_for_high_segbits) {
909 if (single_insn_swpd) {
910 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done);
911 uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
912 did_vmalloc_branch = 1;
913 /* fall through */
914 } else {
915 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault);
916 }
917 }
918 if (!did_vmalloc_branch) {
919 if (single_insn_swpd) {
920 uasm_il_b(p, r, label_vmalloc_done);
921 uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
922 } else {
923 UASM_i_LA_mostly(p, ptr, swpd);
924 uasm_il_b(p, r, label_vmalloc_done);
925 if (uasm_in_compat_space_p(swpd))
926 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
927 else
928 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
929 }
930 }
931 if (mode != not_refill && check_for_high_segbits) {
932 uasm_l_large_segbits_fault(l, *p);
933
934 if (mode == refill_scratch && scratch_reg >= 0)
935 uasm_i_ehb(p);
936
937 /*
938 * We get here if we are an xsseg address, or if we are
939 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary.
940 *
941 * Ignoring xsseg (assume disabled so would generate
942 * (address errors?), the only remaining possibility
943 * is the upper xuseg addresses. On processors with
944 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these
945 * addresses would have taken an address error. We try
946 * to mimic that here by taking a load/istream page
947 * fault.
948 */
949 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS))
950 uasm_i_sync(p, 0);
951 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0);
952 uasm_i_jr(p, ptr);
953
954 if (mode == refill_scratch) {
955 if (scratch_reg >= 0)
956 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
957 else
958 UASM_i_LW(p, 1, scratchpad_offset(0), 0);
959 } else {
960 uasm_i_nop(p);
961 }
962 }
963}
964
965#else /* !CONFIG_64BIT */
966
967/*
968 * TMP and PTR are scratch.
969 * TMP will be clobbered, PTR will hold the pgd entry.
970 */
971void build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
972{
973 if (pgd_reg != -1) {
974 /* pgd is in pgd_reg */
975 uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg);
976 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
977 } else {
978 long pgdc = (long)pgd_current;
979
980 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
981#ifdef CONFIG_SMP
982 uasm_i_mfc0(p, ptr, SMP_CPUID_REG);
983 UASM_i_LA_mostly(p, tmp, pgdc);
984 uasm_i_srl(p, ptr, ptr, SMP_CPUID_PTRSHIFT);
985 uasm_i_addu(p, ptr, tmp, ptr);
986#else
987 UASM_i_LA_mostly(p, ptr, pgdc);
988#endif
989 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
990 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
991 }
992 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
993 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2);
994 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
995}
996EXPORT_SYMBOL_GPL(build_get_pgde32);
997
998#endif /* !CONFIG_64BIT */
999
1000static void build_adjust_context(u32 **p, unsigned int ctx)
1001{
1002 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
1003 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
1004
1005 switch (current_cpu_type()) {
1006 case CPU_VR41XX:
1007 case CPU_VR4111:
1008 case CPU_VR4121:
1009 case CPU_VR4122:
1010 case CPU_VR4131:
1011 case CPU_VR4181:
1012 case CPU_VR4181A:
1013 case CPU_VR4133:
1014 shift += 2;
1015 break;
1016
1017 default:
1018 break;
1019 }
1020
1021 if (shift)
1022 UASM_i_SRL(p, ctx, ctx, shift);
1023 uasm_i_andi(p, ctx, ctx, mask);
1024}
1025
1026void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
1027{
1028 /*
1029 * Bug workaround for the Nevada. It seems as if under certain
1030 * circumstances the move from cp0_context might produce a
1031 * bogus result when the mfc0 instruction and its consumer are
1032 * in a different cacheline or a load instruction, probably any
1033 * memory reference, is between them.
1034 */
1035 switch (current_cpu_type()) {
1036 case CPU_NEVADA:
1037 UASM_i_LW(p, ptr, 0, ptr);
1038 GET_CONTEXT(p, tmp); /* get context reg */
1039 break;
1040
1041 default:
1042 GET_CONTEXT(p, tmp); /* get context reg */
1043 UASM_i_LW(p, ptr, 0, ptr);
1044 break;
1045 }
1046
1047 build_adjust_context(p, tmp);
1048 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
1049}
1050EXPORT_SYMBOL_GPL(build_get_ptep);
1051
1052void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep)
1053{
1054 int pte_off_even = 0;
1055 int pte_off_odd = sizeof(pte_t);
1056
1057#if defined(CONFIG_CPU_MIPS32) && defined(CONFIG_PHYS_ADDR_T_64BIT)
1058 /* The low 32 bits of EntryLo is stored in pte_high */
1059 pte_off_even += offsetof(pte_t, pte_high);
1060 pte_off_odd += offsetof(pte_t, pte_high);
1061#endif
1062
1063 if (IS_ENABLED(CONFIG_XPA)) {
1064 uasm_i_lw(p, tmp, pte_off_even, ptep); /* even pte */
1065 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL));
1066 UASM_i_MTC0(p, tmp, C0_ENTRYLO0);
1067
1068 if (cpu_has_xpa && !mips_xpa_disabled) {
1069 uasm_i_lw(p, tmp, 0, ptep);
1070 uasm_i_ext(p, tmp, tmp, 0, 24);
1071 uasm_i_mthc0(p, tmp, C0_ENTRYLO0);
1072 }
1073
1074 uasm_i_lw(p, tmp, pte_off_odd, ptep); /* odd pte */
1075 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL));
1076 UASM_i_MTC0(p, tmp, C0_ENTRYLO1);
1077
1078 if (cpu_has_xpa && !mips_xpa_disabled) {
1079 uasm_i_lw(p, tmp, sizeof(pte_t), ptep);
1080 uasm_i_ext(p, tmp, tmp, 0, 24);
1081 uasm_i_mthc0(p, tmp, C0_ENTRYLO1);
1082 }
1083 return;
1084 }
1085
1086 UASM_i_LW(p, tmp, pte_off_even, ptep); /* get even pte */
1087 UASM_i_LW(p, ptep, pte_off_odd, ptep); /* get odd pte */
1088 if (r45k_bvahwbug())
1089 build_tlb_probe_entry(p);
1090 build_convert_pte_to_entrylo(p, tmp);
1091 if (r4k_250MHZhwbug())
1092 UASM_i_MTC0(p, 0, C0_ENTRYLO0);
1093 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
1094 build_convert_pte_to_entrylo(p, ptep);
1095 if (r45k_bvahwbug())
1096 uasm_i_mfc0(p, tmp, C0_INDEX);
1097 if (r4k_250MHZhwbug())
1098 UASM_i_MTC0(p, 0, C0_ENTRYLO1);
1099 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
1100}
1101EXPORT_SYMBOL_GPL(build_update_entries);
1102
1103struct mips_huge_tlb_info {
1104 int huge_pte;
1105 int restore_scratch;
1106 bool need_reload_pte;
1107};
1108
1109static struct mips_huge_tlb_info
1110build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
1111 struct uasm_reloc **r, unsigned int tmp,
1112 unsigned int ptr, int c0_scratch_reg)
1113{
1114 struct mips_huge_tlb_info rv;
1115 unsigned int even, odd;
1116 int vmalloc_branch_delay_filled = 0;
1117 const int scratch = 1; /* Our extra working register */
1118
1119 rv.huge_pte = scratch;
1120 rv.restore_scratch = 0;
1121 rv.need_reload_pte = false;
1122
1123 if (check_for_high_segbits) {
1124 UASM_i_MFC0(p, tmp, C0_BADVADDR);
1125
1126 if (pgd_reg != -1)
1127 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
1128 else
1129 UASM_i_MFC0(p, ptr, C0_CONTEXT);
1130
1131 if (c0_scratch_reg >= 0)
1132 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg);
1133 else
1134 UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
1135
1136 uasm_i_dsrl_safe(p, scratch, tmp,
1137 PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
1138 uasm_il_bnez(p, r, scratch, label_vmalloc);
1139
1140 if (pgd_reg == -1) {
1141 vmalloc_branch_delay_filled = 1;
1142 /* Clear lower 23 bits of context. */
1143 uasm_i_dins(p, ptr, 0, 0, 23);
1144 }
1145 } else {
1146 if (pgd_reg != -1)
1147 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
1148 else
1149 UASM_i_MFC0(p, ptr, C0_CONTEXT);
1150
1151 UASM_i_MFC0(p, tmp, C0_BADVADDR);
1152
1153 if (c0_scratch_reg >= 0)
1154 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg);
1155 else
1156 UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
1157
1158 if (pgd_reg == -1)
1159 /* Clear lower 23 bits of context. */
1160 uasm_i_dins(p, ptr, 0, 0, 23);
1161
1162 uasm_il_bltz(p, r, tmp, label_vmalloc);
1163 }
1164
1165 if (pgd_reg == -1) {
1166 vmalloc_branch_delay_filled = 1;
1167 /* 1 0 1 0 1 << 6 xkphys cached */
1168 uasm_i_ori(p, ptr, ptr, 0x540);
1169 uasm_i_drotr(p, ptr, ptr, 11);
1170 }
1171
1172#ifdef __PAGETABLE_PMD_FOLDED
1173#define LOC_PTEP scratch
1174#else
1175#define LOC_PTEP ptr
1176#endif
1177
1178 if (!vmalloc_branch_delay_filled)
1179 /* get pgd offset in bytes */
1180 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
1181
1182 uasm_l_vmalloc_done(l, *p);
1183
1184 /*
1185 * tmp ptr
1186 * fall-through case = badvaddr *pgd_current
1187 * vmalloc case = badvaddr swapper_pg_dir
1188 */
1189
1190 if (vmalloc_branch_delay_filled)
1191 /* get pgd offset in bytes */
1192 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
1193
1194#ifdef __PAGETABLE_PMD_FOLDED
1195 GET_CONTEXT(p, tmp); /* get context reg */
1196#endif
1197 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3);
1198
1199 if (use_lwx_insns()) {
1200 UASM_i_LWX(p, LOC_PTEP, scratch, ptr);
1201 } else {
1202 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */
1203 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */
1204 }
1205
1206#ifndef __PAGETABLE_PUD_FOLDED
1207 /* get pud offset in bytes */
1208 uasm_i_dsrl_safe(p, scratch, tmp, PUD_SHIFT - 3);
1209 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PUD - 1) << 3);
1210
1211 if (use_lwx_insns()) {
1212 UASM_i_LWX(p, ptr, scratch, ptr);
1213 } else {
1214 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */
1215 UASM_i_LW(p, ptr, 0, ptr);
1216 }
1217 /* ptr contains a pointer to PMD entry */
1218 /* tmp contains the address */
1219#endif
1220
1221#ifndef __PAGETABLE_PMD_FOLDED
1222 /* get pmd offset in bytes */
1223 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3);
1224 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3);
1225 GET_CONTEXT(p, tmp); /* get context reg */
1226
1227 if (use_lwx_insns()) {
1228 UASM_i_LWX(p, scratch, scratch, ptr);
1229 } else {
1230 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */
1231 UASM_i_LW(p, scratch, 0, ptr);
1232 }
1233#endif
1234 /* Adjust the context during the load latency. */
1235 build_adjust_context(p, tmp);
1236
1237#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1238 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update);
1239 /*
1240 * The in the LWX case we don't want to do the load in the
1241 * delay slot. It cannot issue in the same cycle and may be
1242 * speculative and unneeded.
1243 */
1244 if (use_lwx_insns())
1245 uasm_i_nop(p);
1246#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */
1247
1248
1249 /* build_update_entries */
1250 if (use_lwx_insns()) {
1251 even = ptr;
1252 odd = tmp;
1253 UASM_i_LWX(p, even, scratch, tmp);
1254 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t));
1255 UASM_i_LWX(p, odd, scratch, tmp);
1256 } else {
1257 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */
1258 even = tmp;
1259 odd = ptr;
1260 UASM_i_LW(p, even, 0, ptr); /* get even pte */
1261 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */
1262 }
1263 if (cpu_has_rixi) {
1264 uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL));
1265 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
1266 uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL));
1267 } else {
1268 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL));
1269 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
1270 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL));
1271 }
1272 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */
1273
1274 if (c0_scratch_reg >= 0) {
1275 uasm_i_ehb(p);
1276 UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg);
1277 build_tlb_write_entry(p, l, r, tlb_random);
1278 uasm_l_leave(l, *p);
1279 rv.restore_scratch = 1;
1280 } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) {
1281 build_tlb_write_entry(p, l, r, tlb_random);
1282 uasm_l_leave(l, *p);
1283 UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
1284 } else {
1285 UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
1286 build_tlb_write_entry(p, l, r, tlb_random);
1287 uasm_l_leave(l, *p);
1288 rv.restore_scratch = 1;
1289 }
1290
1291 uasm_i_eret(p); /* return from trap */
1292
1293 return rv;
1294}
1295
1296/*
1297 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception
1298 * because EXL == 0. If we wrap, we can also use the 32 instruction
1299 * slots before the XTLB refill exception handler which belong to the
1300 * unused TLB refill exception.
1301 */
1302#define MIPS64_REFILL_INSNS 32
1303
1304static void build_r4000_tlb_refill_handler(void)
1305{
1306 u32 *p = tlb_handler;
1307 struct uasm_label *l = labels;
1308 struct uasm_reloc *r = relocs;
1309 u32 *f;
1310 unsigned int final_len;
1311 struct mips_huge_tlb_info htlb_info __maybe_unused;
1312 enum vmalloc64_mode vmalloc_mode __maybe_unused;
1313
1314 memset(tlb_handler, 0, sizeof(tlb_handler));
1315 memset(labels, 0, sizeof(labels));
1316 memset(relocs, 0, sizeof(relocs));
1317 memset(final_handler, 0, sizeof(final_handler));
1318
1319 if (IS_ENABLED(CONFIG_64BIT) && (scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) {
1320 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1,
1321 scratch_reg);
1322 vmalloc_mode = refill_scratch;
1323 } else {
1324 htlb_info.huge_pte = K0;
1325 htlb_info.restore_scratch = 0;
1326 htlb_info.need_reload_pte = true;
1327 vmalloc_mode = refill_noscratch;
1328 /*
1329 * create the plain linear handler
1330 */
1331 if (bcm1250_m3_war()) {
1332 unsigned int segbits = 44;
1333
1334 uasm_i_dmfc0(&p, K0, C0_BADVADDR);
1335 uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
1336 uasm_i_xor(&p, K0, K0, K1);
1337 uasm_i_dsrl_safe(&p, K1, K0, 62);
1338 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
1339 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
1340 uasm_i_or(&p, K0, K0, K1);
1341 uasm_il_bnez(&p, &r, K0, label_leave);
1342 /* No need for uasm_i_nop */
1343 }
1344
1345#ifdef CONFIG_64BIT
1346 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
1347#else
1348 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
1349#endif
1350
1351#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1352 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update);
1353#endif
1354
1355 build_get_ptep(&p, K0, K1);
1356 build_update_entries(&p, K0, K1);
1357 build_tlb_write_entry(&p, &l, &r, tlb_random);
1358 uasm_l_leave(&l, p);
1359 uasm_i_eret(&p); /* return from trap */
1360 }
1361#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1362 uasm_l_tlb_huge_update(&l, p);
1363 if (htlb_info.need_reload_pte)
1364 UASM_i_LW(&p, htlb_info.huge_pte, 0, K1);
1365 build_huge_update_entries(&p, htlb_info.huge_pte, K1);
1366 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random,
1367 htlb_info.restore_scratch);
1368#endif
1369
1370#ifdef CONFIG_64BIT
1371 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode);
1372#endif
1373
1374 /*
1375 * Overflow check: For the 64bit handler, we need at least one
1376 * free instruction slot for the wrap-around branch. In worst
1377 * case, if the intended insertion point is a delay slot, we
1378 * need three, with the second nop'ed and the third being
1379 * unused.
1380 */
1381 switch (boot_cpu_type()) {
1382 default:
1383 if (sizeof(long) == 4) {
1384 case CPU_LOONGSON2EF:
1385 /* Loongson2 ebase is different than r4k, we have more space */
1386 if ((p - tlb_handler) > 64)
1387 panic("TLB refill handler space exceeded");
1388 /*
1389 * Now fold the handler in the TLB refill handler space.
1390 */
1391 f = final_handler;
1392 /* Simplest case, just copy the handler. */
1393 uasm_copy_handler(relocs, labels, tlb_handler, p, f);
1394 final_len = p - tlb_handler;
1395 break;
1396 } else {
1397 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)
1398 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)
1399 && uasm_insn_has_bdelay(relocs,
1400 tlb_handler + MIPS64_REFILL_INSNS - 3)))
1401 panic("TLB refill handler space exceeded");
1402 /*
1403 * Now fold the handler in the TLB refill handler space.
1404 */
1405 f = final_handler + MIPS64_REFILL_INSNS;
1406 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {
1407 /* Just copy the handler. */
1408 uasm_copy_handler(relocs, labels, tlb_handler, p, f);
1409 final_len = p - tlb_handler;
1410 } else {
1411#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1412 const enum label_id ls = label_tlb_huge_update;
1413#else
1414 const enum label_id ls = label_vmalloc;
1415#endif
1416 u32 *split;
1417 int ov = 0;
1418 int i;
1419
1420 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++)
1421 ;
1422 BUG_ON(i == ARRAY_SIZE(labels));
1423 split = labels[i].addr;
1424
1425 /*
1426 * See if we have overflown one way or the other.
1427 */
1428 if (split > tlb_handler + MIPS64_REFILL_INSNS ||
1429 split < p - MIPS64_REFILL_INSNS)
1430 ov = 1;
1431
1432 if (ov) {
1433 /*
1434 * Split two instructions before the end. One
1435 * for the branch and one for the instruction
1436 * in the delay slot.
1437 */
1438 split = tlb_handler + MIPS64_REFILL_INSNS - 2;
1439
1440 /*
1441 * If the branch would fall in a delay slot,
1442 * we must back up an additional instruction
1443 * so that it is no longer in a delay slot.
1444 */
1445 if (uasm_insn_has_bdelay(relocs, split - 1))
1446 split--;
1447 }
1448 /* Copy first part of the handler. */
1449 uasm_copy_handler(relocs, labels, tlb_handler, split, f);
1450 f += split - tlb_handler;
1451
1452 if (ov) {
1453 /* Insert branch. */
1454 uasm_l_split(&l, final_handler);
1455 uasm_il_b(&f, &r, label_split);
1456 if (uasm_insn_has_bdelay(relocs, split))
1457 uasm_i_nop(&f);
1458 else {
1459 uasm_copy_handler(relocs, labels,
1460 split, split + 1, f);
1461 uasm_move_labels(labels, f, f + 1, -1);
1462 f++;
1463 split++;
1464 }
1465 }
1466
1467 /* Copy the rest of the handler. */
1468 uasm_copy_handler(relocs, labels, split, p, final_handler);
1469 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +
1470 (p - split);
1471 }
1472 }
1473 break;
1474 }
1475
1476 uasm_resolve_relocs(relocs, labels);
1477 pr_debug("Wrote TLB refill handler (%u instructions).\n",
1478 final_len);
1479
1480 memcpy((void *)ebase, final_handler, 0x100);
1481 local_flush_icache_range(ebase, ebase + 0x100);
1482 dump_handler("r4000_tlb_refill", (u32 *)ebase, (u32 *)(ebase + 0x100));
1483}
1484
1485static void setup_pw(void)
1486{
1487 unsigned int pwctl;
1488 unsigned long pgd_i, pgd_w;
1489#ifndef __PAGETABLE_PMD_FOLDED
1490 unsigned long pmd_i, pmd_w;
1491#endif
1492 unsigned long pt_i, pt_w;
1493 unsigned long pte_i, pte_w;
1494#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1495 unsigned long psn;
1496
1497 psn = ilog2(_PAGE_HUGE); /* bit used to indicate huge page */
1498#endif
1499 pgd_i = PGDIR_SHIFT; /* 1st level PGD */
1500#ifndef __PAGETABLE_PMD_FOLDED
1501 pgd_w = PGDIR_SHIFT - PMD_SHIFT + PGD_ORDER;
1502
1503 pmd_i = PMD_SHIFT; /* 2nd level PMD */
1504 pmd_w = PMD_SHIFT - PAGE_SHIFT;
1505#else
1506 pgd_w = PGDIR_SHIFT - PAGE_SHIFT + PGD_ORDER;
1507#endif
1508
1509 pt_i = PAGE_SHIFT; /* 3rd level PTE */
1510 pt_w = PAGE_SHIFT - 3;
1511
1512 pte_i = ilog2(_PAGE_GLOBAL);
1513 pte_w = 0;
1514 pwctl = 1 << 30; /* Set PWDirExt */
1515
1516#ifndef __PAGETABLE_PMD_FOLDED
1517 write_c0_pwfield(pgd_i << 24 | pmd_i << 12 | pt_i << 6 | pte_i);
1518 write_c0_pwsize(1 << 30 | pgd_w << 24 | pmd_w << 12 | pt_w << 6 | pte_w);
1519#else
1520 write_c0_pwfield(pgd_i << 24 | pt_i << 6 | pte_i);
1521 write_c0_pwsize(1 << 30 | pgd_w << 24 | pt_w << 6 | pte_w);
1522#endif
1523
1524#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1525 pwctl |= (1 << 6 | psn);
1526#endif
1527 write_c0_pwctl(pwctl);
1528 write_c0_kpgd((long)swapper_pg_dir);
1529 kscratch_used_mask |= (1 << 7); /* KScratch6 is used for KPGD */
1530}
1531
1532static void build_loongson3_tlb_refill_handler(void)
1533{
1534 u32 *p = tlb_handler;
1535 struct uasm_label *l = labels;
1536 struct uasm_reloc *r = relocs;
1537
1538 memset(labels, 0, sizeof(labels));
1539 memset(relocs, 0, sizeof(relocs));
1540 memset(tlb_handler, 0, sizeof(tlb_handler));
1541
1542 if (check_for_high_segbits) {
1543 uasm_i_dmfc0(&p, K0, C0_BADVADDR);
1544 uasm_i_dsrl_safe(&p, K1, K0, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
1545 uasm_il_beqz(&p, &r, K1, label_vmalloc);
1546 uasm_i_nop(&p);
1547
1548 uasm_il_bgez(&p, &r, K0, label_large_segbits_fault);
1549 uasm_i_nop(&p);
1550 uasm_l_vmalloc(&l, p);
1551 }
1552
1553 uasm_i_dmfc0(&p, K1, C0_PGD);
1554
1555 uasm_i_lddir(&p, K0, K1, 3); /* global page dir */
1556#ifndef __PAGETABLE_PMD_FOLDED
1557 uasm_i_lddir(&p, K1, K0, 1); /* middle page dir */
1558#endif
1559 uasm_i_ldpte(&p, K1, 0); /* even */
1560 uasm_i_ldpte(&p, K1, 1); /* odd */
1561 uasm_i_tlbwr(&p);
1562
1563 /* restore page mask */
1564 if (PM_DEFAULT_MASK >> 16) {
1565 uasm_i_lui(&p, K0, PM_DEFAULT_MASK >> 16);
1566 uasm_i_ori(&p, K0, K0, PM_DEFAULT_MASK & 0xffff);
1567 uasm_i_mtc0(&p, K0, C0_PAGEMASK);
1568 } else if (PM_DEFAULT_MASK) {
1569 uasm_i_ori(&p, K0, 0, PM_DEFAULT_MASK);
1570 uasm_i_mtc0(&p, K0, C0_PAGEMASK);
1571 } else {
1572 uasm_i_mtc0(&p, 0, C0_PAGEMASK);
1573 }
1574
1575 uasm_i_eret(&p);
1576
1577 if (check_for_high_segbits) {
1578 uasm_l_large_segbits_fault(&l, p);
1579 UASM_i_LA(&p, K1, (unsigned long)tlb_do_page_fault_0);
1580 uasm_i_jr(&p, K1);
1581 uasm_i_nop(&p);
1582 }
1583
1584 uasm_resolve_relocs(relocs, labels);
1585 memcpy((void *)(ebase + 0x80), tlb_handler, 0x80);
1586 local_flush_icache_range(ebase + 0x80, ebase + 0x100);
1587 dump_handler("loongson3_tlb_refill",
1588 (u32 *)(ebase + 0x80), (u32 *)(ebase + 0x100));
1589}
1590
1591static void build_setup_pgd(void)
1592{
1593 const int a0 = 4;
1594 const int __maybe_unused a1 = 5;
1595 const int __maybe_unused a2 = 6;
1596 u32 *p = (u32 *)msk_isa16_mode((ulong)tlbmiss_handler_setup_pgd);
1597#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
1598 long pgdc = (long)pgd_current;
1599#endif
1600
1601 memset(p, 0, tlbmiss_handler_setup_pgd_end - (char *)p);
1602 memset(labels, 0, sizeof(labels));
1603 memset(relocs, 0, sizeof(relocs));
1604 pgd_reg = allocate_kscratch();
1605#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
1606 if (pgd_reg == -1) {
1607 struct uasm_label *l = labels;
1608 struct uasm_reloc *r = relocs;
1609
1610 /* PGD << 11 in c0_Context */
1611 /*
1612 * If it is a ckseg0 address, convert to a physical
1613 * address. Shifting right by 29 and adding 4 will
1614 * result in zero for these addresses.
1615 *
1616 */
1617 UASM_i_SRA(&p, a1, a0, 29);
1618 UASM_i_ADDIU(&p, a1, a1, 4);
1619 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1);
1620 uasm_i_nop(&p);
1621 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29);
1622 uasm_l_tlbl_goaround1(&l, p);
1623 UASM_i_SLL(&p, a0, a0, 11);
1624 UASM_i_MTC0(&p, a0, C0_CONTEXT);
1625 uasm_i_jr(&p, 31);
1626 uasm_i_ehb(&p);
1627 } else {
1628 /* PGD in c0_KScratch */
1629 if (cpu_has_ldpte)
1630 UASM_i_MTC0(&p, a0, C0_PWBASE);
1631 else
1632 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
1633 uasm_i_jr(&p, 31);
1634 uasm_i_ehb(&p);
1635 }
1636#else
1637#ifdef CONFIG_SMP
1638 /* Save PGD to pgd_current[smp_processor_id()] */
1639 UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG);
1640 UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT);
1641 UASM_i_LA_mostly(&p, a2, pgdc);
1642 UASM_i_ADDU(&p, a2, a2, a1);
1643 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
1644#else
1645 UASM_i_LA_mostly(&p, a2, pgdc);
1646 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
1647#endif /* SMP */
1648
1649 /* if pgd_reg is allocated, save PGD also to scratch register */
1650 if (pgd_reg != -1) {
1651 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
1652 uasm_i_jr(&p, 31);
1653 uasm_i_ehb(&p);
1654 } else {
1655 uasm_i_jr(&p, 31);
1656 uasm_i_nop(&p);
1657 }
1658#endif
1659 if (p >= (u32 *)tlbmiss_handler_setup_pgd_end)
1660 panic("tlbmiss_handler_setup_pgd space exceeded");
1661
1662 uasm_resolve_relocs(relocs, labels);
1663 pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n",
1664 (unsigned int)(p - (u32 *)tlbmiss_handler_setup_pgd));
1665
1666 dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd,
1667 tlbmiss_handler_setup_pgd_end);
1668}
1669
1670static void
1671iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
1672{
1673#ifdef CONFIG_SMP
1674 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS))
1675 uasm_i_sync(p, 0);
1676# ifdef CONFIG_PHYS_ADDR_T_64BIT
1677 if (cpu_has_64bits)
1678 uasm_i_lld(p, pte, 0, ptr);
1679 else
1680# endif
1681 UASM_i_LL(p, pte, 0, ptr);
1682#else
1683# ifdef CONFIG_PHYS_ADDR_T_64BIT
1684 if (cpu_has_64bits)
1685 uasm_i_ld(p, pte, 0, ptr);
1686 else
1687# endif
1688 UASM_i_LW(p, pte, 0, ptr);
1689#endif
1690}
1691
1692static void
1693iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
1694 unsigned int mode, unsigned int scratch)
1695{
1696 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
1697 unsigned int swmode = mode & ~hwmode;
1698
1699 if (IS_ENABLED(CONFIG_XPA) && !cpu_has_64bits) {
1700 uasm_i_lui(p, scratch, swmode >> 16);
1701 uasm_i_or(p, pte, pte, scratch);
1702 BUG_ON(swmode & 0xffff);
1703 } else {
1704 uasm_i_ori(p, pte, pte, mode);
1705 }
1706
1707#ifdef CONFIG_SMP
1708# ifdef CONFIG_PHYS_ADDR_T_64BIT
1709 if (cpu_has_64bits)
1710 uasm_i_scd(p, pte, 0, ptr);
1711 else
1712# endif
1713 UASM_i_SC(p, pte, 0, ptr);
1714
1715 if (r10000_llsc_war())
1716 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change);
1717 else
1718 uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
1719
1720# ifdef CONFIG_PHYS_ADDR_T_64BIT
1721 if (!cpu_has_64bits) {
1722 /* no uasm_i_nop needed */
1723 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr);
1724 uasm_i_ori(p, pte, pte, hwmode);
1725 BUG_ON(hwmode & ~0xffff);
1726 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr);
1727 uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
1728 /* no uasm_i_nop needed */
1729 uasm_i_lw(p, pte, 0, ptr);
1730 } else
1731 uasm_i_nop(p);
1732# else
1733 uasm_i_nop(p);
1734# endif
1735#else
1736# ifdef CONFIG_PHYS_ADDR_T_64BIT
1737 if (cpu_has_64bits)
1738 uasm_i_sd(p, pte, 0, ptr);
1739 else
1740# endif
1741 UASM_i_SW(p, pte, 0, ptr);
1742
1743# ifdef CONFIG_PHYS_ADDR_T_64BIT
1744 if (!cpu_has_64bits) {
1745 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr);
1746 uasm_i_ori(p, pte, pte, hwmode);
1747 BUG_ON(hwmode & ~0xffff);
1748 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr);
1749 uasm_i_lw(p, pte, 0, ptr);
1750 }
1751# endif
1752#endif
1753}
1754
1755/*
1756 * Check if PTE is present, if not then jump to LABEL. PTR points to
1757 * the page table where this PTE is located, PTE will be re-loaded
1758 * with it's original value.
1759 */
1760static void
1761build_pte_present(u32 **p, struct uasm_reloc **r,
1762 int pte, int ptr, int scratch, enum label_id lid)
1763{
1764 int t = scratch >= 0 ? scratch : pte;
1765 int cur = pte;
1766
1767 if (cpu_has_rixi) {
1768 if (use_bbit_insns()) {
1769 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
1770 uasm_i_nop(p);
1771 } else {
1772 if (_PAGE_PRESENT_SHIFT) {
1773 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT);
1774 cur = t;
1775 }
1776 uasm_i_andi(p, t, cur, 1);
1777 uasm_il_beqz(p, r, t, lid);
1778 if (pte == t)
1779 /* You lose the SMP race :-(*/
1780 iPTE_LW(p, pte, ptr);
1781 }
1782 } else {
1783 if (_PAGE_PRESENT_SHIFT) {
1784 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT);
1785 cur = t;
1786 }
1787 uasm_i_andi(p, t, cur,
1788 (_PAGE_PRESENT | _PAGE_NO_READ) >> _PAGE_PRESENT_SHIFT);
1789 uasm_i_xori(p, t, t, _PAGE_PRESENT >> _PAGE_PRESENT_SHIFT);
1790 uasm_il_bnez(p, r, t, lid);
1791 if (pte == t)
1792 /* You lose the SMP race :-(*/
1793 iPTE_LW(p, pte, ptr);
1794 }
1795}
1796
1797/* Make PTE valid, store result in PTR. */
1798static void
1799build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
1800 unsigned int ptr, unsigned int scratch)
1801{
1802 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
1803
1804 iPTE_SW(p, r, pte, ptr, mode, scratch);
1805}
1806
1807/*
1808 * Check if PTE can be written to, if not branch to LABEL. Regardless
1809 * restore PTE with value from PTR when done.
1810 */
1811static void
1812build_pte_writable(u32 **p, struct uasm_reloc **r,
1813 unsigned int pte, unsigned int ptr, int scratch,
1814 enum label_id lid)
1815{
1816 int t = scratch >= 0 ? scratch : pte;
1817 int cur = pte;
1818
1819 if (_PAGE_PRESENT_SHIFT) {
1820 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT);
1821 cur = t;
1822 }
1823 uasm_i_andi(p, t, cur,
1824 (_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT);
1825 uasm_i_xori(p, t, t,
1826 (_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT);
1827 uasm_il_bnez(p, r, t, lid);
1828 if (pte == t)
1829 /* You lose the SMP race :-(*/
1830 iPTE_LW(p, pte, ptr);
1831 else
1832 uasm_i_nop(p);
1833}
1834
1835/* Make PTE writable, update software status bits as well, then store
1836 * at PTR.
1837 */
1838static void
1839build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
1840 unsigned int ptr, unsigned int scratch)
1841{
1842 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
1843 | _PAGE_DIRTY);
1844
1845 iPTE_SW(p, r, pte, ptr, mode, scratch);
1846}
1847
1848/*
1849 * Check if PTE can be modified, if not branch to LABEL. Regardless
1850 * restore PTE with value from PTR when done.
1851 */
1852static void
1853build_pte_modifiable(u32 **p, struct uasm_reloc **r,
1854 unsigned int pte, unsigned int ptr, int scratch,
1855 enum label_id lid)
1856{
1857 if (use_bbit_insns()) {
1858 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
1859 uasm_i_nop(p);
1860 } else {
1861 int t = scratch >= 0 ? scratch : pte;
1862 uasm_i_srl(p, t, pte, _PAGE_WRITE_SHIFT);
1863 uasm_i_andi(p, t, t, 1);
1864 uasm_il_beqz(p, r, t, lid);
1865 if (pte == t)
1866 /* You lose the SMP race :-(*/
1867 iPTE_LW(p, pte, ptr);
1868 }
1869}
1870
1871#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
1872
1873
1874/*
1875 * R3000 style TLB load/store/modify handlers.
1876 */
1877
1878/*
1879 * This places the pte into ENTRYLO0 and writes it with tlbwi.
1880 * Then it returns.
1881 */
1882static void
1883build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
1884{
1885 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1886 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
1887 uasm_i_tlbwi(p);
1888 uasm_i_jr(p, tmp);
1889 uasm_i_rfe(p); /* branch delay */
1890}
1891
1892/*
1893 * This places the pte into ENTRYLO0 and writes it with tlbwi
1894 * or tlbwr as appropriate. This is because the index register
1895 * may have the probe fail bit set as a result of a trap on a
1896 * kseg2 access, i.e. without refill. Then it returns.
1897 */
1898static void
1899build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
1900 struct uasm_reloc **r, unsigned int pte,
1901 unsigned int tmp)
1902{
1903 uasm_i_mfc0(p, tmp, C0_INDEX);
1904 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1905 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
1906 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */
1907 uasm_i_tlbwi(p); /* cp0 delay */
1908 uasm_i_jr(p, tmp);
1909 uasm_i_rfe(p); /* branch delay */
1910 uasm_l_r3000_write_probe_fail(l, *p);
1911 uasm_i_tlbwr(p); /* cp0 delay */
1912 uasm_i_jr(p, tmp);
1913 uasm_i_rfe(p); /* branch delay */
1914}
1915
1916static void
1917build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
1918 unsigned int ptr)
1919{
1920 long pgdc = (long)pgd_current;
1921
1922 uasm_i_mfc0(p, pte, C0_BADVADDR);
1923 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */
1924 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
1925 uasm_i_srl(p, pte, pte, 22); /* load delay */
1926 uasm_i_sll(p, pte, pte, 2);
1927 uasm_i_addu(p, ptr, ptr, pte);
1928 uasm_i_mfc0(p, pte, C0_CONTEXT);
1929 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */
1930 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */
1931 uasm_i_addu(p, ptr, ptr, pte);
1932 uasm_i_lw(p, pte, 0, ptr);
1933 uasm_i_tlbp(p); /* load delay */
1934}
1935
1936static void build_r3000_tlb_load_handler(void)
1937{
1938 u32 *p = (u32 *)handle_tlbl;
1939 struct uasm_label *l = labels;
1940 struct uasm_reloc *r = relocs;
1941
1942 memset(p, 0, handle_tlbl_end - (char *)p);
1943 memset(labels, 0, sizeof(labels));
1944 memset(relocs, 0, sizeof(relocs));
1945
1946 build_r3000_tlbchange_handler_head(&p, K0, K1);
1947 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl);
1948 uasm_i_nop(&p); /* load delay */
1949 build_make_valid(&p, &r, K0, K1, -1);
1950 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1951
1952 uasm_l_nopage_tlbl(&l, p);
1953 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1954 uasm_i_nop(&p);
1955
1956 if (p >= (u32 *)handle_tlbl_end)
1957 panic("TLB load handler fastpath space exceeded");
1958
1959 uasm_resolve_relocs(relocs, labels);
1960 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
1961 (unsigned int)(p - (u32 *)handle_tlbl));
1962
1963 dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_end);
1964}
1965
1966static void build_r3000_tlb_store_handler(void)
1967{
1968 u32 *p = (u32 *)handle_tlbs;
1969 struct uasm_label *l = labels;
1970 struct uasm_reloc *r = relocs;
1971
1972 memset(p, 0, handle_tlbs_end - (char *)p);
1973 memset(labels, 0, sizeof(labels));
1974 memset(relocs, 0, sizeof(relocs));
1975
1976 build_r3000_tlbchange_handler_head(&p, K0, K1);
1977 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs);
1978 uasm_i_nop(&p); /* load delay */
1979 build_make_write(&p, &r, K0, K1, -1);
1980 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1981
1982 uasm_l_nopage_tlbs(&l, p);
1983 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1984 uasm_i_nop(&p);
1985
1986 if (p >= (u32 *)handle_tlbs_end)
1987 panic("TLB store handler fastpath space exceeded");
1988
1989 uasm_resolve_relocs(relocs, labels);
1990 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
1991 (unsigned int)(p - (u32 *)handle_tlbs));
1992
1993 dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_end);
1994}
1995
1996static void build_r3000_tlb_modify_handler(void)
1997{
1998 u32 *p = (u32 *)handle_tlbm;
1999 struct uasm_label *l = labels;
2000 struct uasm_reloc *r = relocs;
2001
2002 memset(p, 0, handle_tlbm_end - (char *)p);
2003 memset(labels, 0, sizeof(labels));
2004 memset(relocs, 0, sizeof(relocs));
2005
2006 build_r3000_tlbchange_handler_head(&p, K0, K1);
2007 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm);
2008 uasm_i_nop(&p); /* load delay */
2009 build_make_write(&p, &r, K0, K1, -1);
2010 build_r3000_pte_reload_tlbwi(&p, K0, K1);
2011
2012 uasm_l_nopage_tlbm(&l, p);
2013 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
2014 uasm_i_nop(&p);
2015
2016 if (p >= (u32 *)handle_tlbm_end)
2017 panic("TLB modify handler fastpath space exceeded");
2018
2019 uasm_resolve_relocs(relocs, labels);
2020 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
2021 (unsigned int)(p - (u32 *)handle_tlbm));
2022
2023 dump_handler("r3000_tlb_modify", handle_tlbm, handle_tlbm_end);
2024}
2025#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
2026
2027static bool cpu_has_tlbex_tlbp_race(void)
2028{
2029 /*
2030 * When a Hardware Table Walker is running it can replace TLB entries
2031 * at any time, leading to a race between it & the CPU.
2032 */
2033 if (cpu_has_htw)
2034 return true;
2035
2036 /*
2037 * If the CPU shares FTLB RAM with its siblings then our entry may be
2038 * replaced at any time by a sibling performing a write to the FTLB.
2039 */
2040 if (cpu_has_shared_ftlb_ram)
2041 return true;
2042
2043 /* In all other cases there ought to be no race condition to handle */
2044 return false;
2045}
2046
2047/*
2048 * R4000 style TLB load/store/modify handlers.
2049 */
2050static struct work_registers
2051build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
2052 struct uasm_reloc **r)
2053{
2054 struct work_registers wr = build_get_work_registers(p);
2055
2056#ifdef CONFIG_64BIT
2057 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */
2058#else
2059 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */
2060#endif
2061
2062#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
2063 /*
2064 * For huge tlb entries, pmd doesn't contain an address but
2065 * instead contains the tlb pte. Check the PAGE_HUGE bit and
2066 * see if we need to jump to huge tlb processing.
2067 */
2068 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update);
2069#endif
2070
2071 UASM_i_MFC0(p, wr.r1, C0_BADVADDR);
2072 UASM_i_LW(p, wr.r2, 0, wr.r2);
2073 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
2074 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
2075 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1);
2076
2077#ifdef CONFIG_SMP
2078 uasm_l_smp_pgtable_change(l, *p);
2079#endif
2080 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */
2081 if (!m4kc_tlbp_war()) {
2082 build_tlb_probe_entry(p);
2083 if (cpu_has_tlbex_tlbp_race()) {
2084 /* race condition happens, leaving */
2085 uasm_i_ehb(p);
2086 uasm_i_mfc0(p, wr.r3, C0_INDEX);
2087 uasm_il_bltz(p, r, wr.r3, label_leave);
2088 uasm_i_nop(p);
2089 }
2090 }
2091 return wr;
2092}
2093
2094static void
2095build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
2096 struct uasm_reloc **r, unsigned int tmp,
2097 unsigned int ptr)
2098{
2099 uasm_i_ori(p, ptr, ptr, sizeof(pte_t));
2100 uasm_i_xori(p, ptr, ptr, sizeof(pte_t));
2101 build_update_entries(p, tmp, ptr);
2102 build_tlb_write_entry(p, l, r, tlb_indexed);
2103 uasm_l_leave(l, *p);
2104 build_restore_work_registers(p);
2105 uasm_i_eret(p); /* return from trap */
2106
2107#ifdef CONFIG_64BIT
2108 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill);
2109#endif
2110}
2111
2112static void build_r4000_tlb_load_handler(void)
2113{
2114 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbl);
2115 struct uasm_label *l = labels;
2116 struct uasm_reloc *r = relocs;
2117 struct work_registers wr;
2118
2119 memset(p, 0, handle_tlbl_end - (char *)p);
2120 memset(labels, 0, sizeof(labels));
2121 memset(relocs, 0, sizeof(relocs));
2122
2123 if (bcm1250_m3_war()) {
2124 unsigned int segbits = 44;
2125
2126 uasm_i_dmfc0(&p, K0, C0_BADVADDR);
2127 uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
2128 uasm_i_xor(&p, K0, K0, K1);
2129 uasm_i_dsrl_safe(&p, K1, K0, 62);
2130 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
2131 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
2132 uasm_i_or(&p, K0, K0, K1);
2133 uasm_il_bnez(&p, &r, K0, label_leave);
2134 /* No need for uasm_i_nop */
2135 }
2136
2137 wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
2138 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
2139 if (m4kc_tlbp_war())
2140 build_tlb_probe_entry(&p);
2141
2142 if (cpu_has_rixi && !cpu_has_rixiex) {
2143 /*
2144 * If the page is not _PAGE_VALID, RI or XI could not
2145 * have triggered it. Skip the expensive test..
2146 */
2147 if (use_bbit_insns()) {
2148 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
2149 label_tlbl_goaround1);
2150 } else {
2151 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
2152 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1);
2153 }
2154 uasm_i_nop(&p);
2155
2156 /*
2157 * Warn if something may race with us & replace the TLB entry
2158 * before we read it here. Everything with such races should
2159 * also have dedicated RiXi exception handlers, so this
2160 * shouldn't be hit.
2161 */
2162 WARN(cpu_has_tlbex_tlbp_race(), "Unhandled race in RiXi path");
2163
2164 uasm_i_tlbr(&p);
2165
2166 switch (current_cpu_type()) {
2167 default:
2168 if (cpu_has_mips_r2_exec_hazard) {
2169 uasm_i_ehb(&p);
2170
2171 case CPU_CAVIUM_OCTEON:
2172 case CPU_CAVIUM_OCTEON_PLUS:
2173 case CPU_CAVIUM_OCTEON2:
2174 break;
2175 }
2176 }
2177
2178 /* Examine entrylo 0 or 1 based on ptr. */
2179 if (use_bbit_insns()) {
2180 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
2181 } else {
2182 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
2183 uasm_i_beqz(&p, wr.r3, 8);
2184 }
2185 /* load it in the delay slot*/
2186 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
2187 /* load it if ptr is odd */
2188 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
2189 /*
2190 * If the entryLo (now in wr.r3) is valid (bit 1), RI or
2191 * XI must have triggered it.
2192 */
2193 if (use_bbit_insns()) {
2194 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl);
2195 uasm_i_nop(&p);
2196 uasm_l_tlbl_goaround1(&l, p);
2197 } else {
2198 uasm_i_andi(&p, wr.r3, wr.r3, 2);
2199 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl);
2200 uasm_i_nop(&p);
2201 }
2202 uasm_l_tlbl_goaround1(&l, p);
2203 }
2204 build_make_valid(&p, &r, wr.r1, wr.r2, wr.r3);
2205 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
2206
2207#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
2208 /*
2209 * This is the entry point when build_r4000_tlbchange_handler_head
2210 * spots a huge page.
2211 */
2212 uasm_l_tlb_huge_update(&l, p);
2213 iPTE_LW(&p, wr.r1, wr.r2);
2214 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
2215 build_tlb_probe_entry(&p);
2216
2217 if (cpu_has_rixi && !cpu_has_rixiex) {
2218 /*
2219 * If the page is not _PAGE_VALID, RI or XI could not
2220 * have triggered it. Skip the expensive test..
2221 */
2222 if (use_bbit_insns()) {
2223 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
2224 label_tlbl_goaround2);
2225 } else {
2226 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
2227 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
2228 }
2229 uasm_i_nop(&p);
2230
2231 /*
2232 * Warn if something may race with us & replace the TLB entry
2233 * before we read it here. Everything with such races should
2234 * also have dedicated RiXi exception handlers, so this
2235 * shouldn't be hit.
2236 */
2237 WARN(cpu_has_tlbex_tlbp_race(), "Unhandled race in RiXi path");
2238
2239 uasm_i_tlbr(&p);
2240
2241 switch (current_cpu_type()) {
2242 default:
2243 if (cpu_has_mips_r2_exec_hazard) {
2244 uasm_i_ehb(&p);
2245
2246 case CPU_CAVIUM_OCTEON:
2247 case CPU_CAVIUM_OCTEON_PLUS:
2248 case CPU_CAVIUM_OCTEON2:
2249 break;
2250 }
2251 }
2252
2253 /* Examine entrylo 0 or 1 based on ptr. */
2254 if (use_bbit_insns()) {
2255 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
2256 } else {
2257 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
2258 uasm_i_beqz(&p, wr.r3, 8);
2259 }
2260 /* load it in the delay slot*/
2261 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
2262 /* load it if ptr is odd */
2263 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
2264 /*
2265 * If the entryLo (now in wr.r3) is valid (bit 1), RI or
2266 * XI must have triggered it.
2267 */
2268 if (use_bbit_insns()) {
2269 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2);
2270 } else {
2271 uasm_i_andi(&p, wr.r3, wr.r3, 2);
2272 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
2273 }
2274 if (PM_DEFAULT_MASK == 0)
2275 uasm_i_nop(&p);
2276 /*
2277 * We clobbered C0_PAGEMASK, restore it. On the other branch
2278 * it is restored in build_huge_tlb_write_entry.
2279 */
2280 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0);
2281
2282 uasm_l_tlbl_goaround2(&l, p);
2283 }
2284 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID));
2285 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1);
2286#endif
2287
2288 uasm_l_nopage_tlbl(&l, p);
2289 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS))
2290 uasm_i_sync(&p, 0);
2291 build_restore_work_registers(&p);
2292#ifdef CONFIG_CPU_MICROMIPS
2293 if ((unsigned long)tlb_do_page_fault_0 & 1) {
2294 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0));
2295 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0));
2296 uasm_i_jr(&p, K0);
2297 } else
2298#endif
2299 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
2300 uasm_i_nop(&p);
2301
2302 if (p >= (u32 *)handle_tlbl_end)
2303 panic("TLB load handler fastpath space exceeded");
2304
2305 uasm_resolve_relocs(relocs, labels);
2306 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
2307 (unsigned int)(p - (u32 *)handle_tlbl));
2308
2309 dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_end);
2310}
2311
2312static void build_r4000_tlb_store_handler(void)
2313{
2314 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbs);
2315 struct uasm_label *l = labels;
2316 struct uasm_reloc *r = relocs;
2317 struct work_registers wr;
2318
2319 memset(p, 0, handle_tlbs_end - (char *)p);
2320 memset(labels, 0, sizeof(labels));
2321 memset(relocs, 0, sizeof(relocs));
2322
2323 wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
2324 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
2325 if (m4kc_tlbp_war())
2326 build_tlb_probe_entry(&p);
2327 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3);
2328 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
2329
2330#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
2331 /*
2332 * This is the entry point when
2333 * build_r4000_tlbchange_handler_head spots a huge page.
2334 */
2335 uasm_l_tlb_huge_update(&l, p);
2336 iPTE_LW(&p, wr.r1, wr.r2);
2337 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
2338 build_tlb_probe_entry(&p);
2339 uasm_i_ori(&p, wr.r1, wr.r1,
2340 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
2341 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1);
2342#endif
2343
2344 uasm_l_nopage_tlbs(&l, p);
2345 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS))
2346 uasm_i_sync(&p, 0);
2347 build_restore_work_registers(&p);
2348#ifdef CONFIG_CPU_MICROMIPS
2349 if ((unsigned long)tlb_do_page_fault_1 & 1) {
2350 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1));
2351 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1));
2352 uasm_i_jr(&p, K0);
2353 } else
2354#endif
2355 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
2356 uasm_i_nop(&p);
2357
2358 if (p >= (u32 *)handle_tlbs_end)
2359 panic("TLB store handler fastpath space exceeded");
2360
2361 uasm_resolve_relocs(relocs, labels);
2362 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
2363 (unsigned int)(p - (u32 *)handle_tlbs));
2364
2365 dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_end);
2366}
2367
2368static void build_r4000_tlb_modify_handler(void)
2369{
2370 u32 *p = (u32 *)msk_isa16_mode((ulong)handle_tlbm);
2371 struct uasm_label *l = labels;
2372 struct uasm_reloc *r = relocs;
2373 struct work_registers wr;
2374
2375 memset(p, 0, handle_tlbm_end - (char *)p);
2376 memset(labels, 0, sizeof(labels));
2377 memset(relocs, 0, sizeof(relocs));
2378
2379 wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
2380 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm);
2381 if (m4kc_tlbp_war())
2382 build_tlb_probe_entry(&p);
2383 /* Present and writable bits set, set accessed and dirty bits. */
2384 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3);
2385 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
2386
2387#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
2388 /*
2389 * This is the entry point when
2390 * build_r4000_tlbchange_handler_head spots a huge page.
2391 */
2392 uasm_l_tlb_huge_update(&l, p);
2393 iPTE_LW(&p, wr.r1, wr.r2);
2394 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm);
2395 build_tlb_probe_entry(&p);
2396 uasm_i_ori(&p, wr.r1, wr.r1,
2397 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
2398 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 0);
2399#endif
2400
2401 uasm_l_nopage_tlbm(&l, p);
2402 if (IS_ENABLED(CONFIG_CPU_LOONGSON3_WORKAROUNDS))
2403 uasm_i_sync(&p, 0);
2404 build_restore_work_registers(&p);
2405#ifdef CONFIG_CPU_MICROMIPS
2406 if ((unsigned long)tlb_do_page_fault_1 & 1) {
2407 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1));
2408 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1));
2409 uasm_i_jr(&p, K0);
2410 } else
2411#endif
2412 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
2413 uasm_i_nop(&p);
2414
2415 if (p >= (u32 *)handle_tlbm_end)
2416 panic("TLB modify handler fastpath space exceeded");
2417
2418 uasm_resolve_relocs(relocs, labels);
2419 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
2420 (unsigned int)(p - (u32 *)handle_tlbm));
2421
2422 dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_end);
2423}
2424
2425static void flush_tlb_handlers(void)
2426{
2427 local_flush_icache_range((unsigned long)handle_tlbl,
2428 (unsigned long)handle_tlbl_end);
2429 local_flush_icache_range((unsigned long)handle_tlbs,
2430 (unsigned long)handle_tlbs_end);
2431 local_flush_icache_range((unsigned long)handle_tlbm,
2432 (unsigned long)handle_tlbm_end);
2433 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
2434 (unsigned long)tlbmiss_handler_setup_pgd_end);
2435}
2436
2437static void print_htw_config(void)
2438{
2439 unsigned long config;
2440 unsigned int pwctl;
2441 const int field = 2 * sizeof(unsigned long);
2442
2443 config = read_c0_pwfield();
2444 pr_debug("PWField (0x%0*lx): GDI: 0x%02lx UDI: 0x%02lx MDI: 0x%02lx PTI: 0x%02lx PTEI: 0x%02lx\n",
2445 field, config,
2446 (config & MIPS_PWFIELD_GDI_MASK) >> MIPS_PWFIELD_GDI_SHIFT,
2447 (config & MIPS_PWFIELD_UDI_MASK) >> MIPS_PWFIELD_UDI_SHIFT,
2448 (config & MIPS_PWFIELD_MDI_MASK) >> MIPS_PWFIELD_MDI_SHIFT,
2449 (config & MIPS_PWFIELD_PTI_MASK) >> MIPS_PWFIELD_PTI_SHIFT,
2450 (config & MIPS_PWFIELD_PTEI_MASK) >> MIPS_PWFIELD_PTEI_SHIFT);
2451
2452 config = read_c0_pwsize();
2453 pr_debug("PWSize (0x%0*lx): PS: 0x%lx GDW: 0x%02lx UDW: 0x%02lx MDW: 0x%02lx PTW: 0x%02lx PTEW: 0x%02lx\n",
2454 field, config,
2455 (config & MIPS_PWSIZE_PS_MASK) >> MIPS_PWSIZE_PS_SHIFT,
2456 (config & MIPS_PWSIZE_GDW_MASK) >> MIPS_PWSIZE_GDW_SHIFT,
2457 (config & MIPS_PWSIZE_UDW_MASK) >> MIPS_PWSIZE_UDW_SHIFT,
2458 (config & MIPS_PWSIZE_MDW_MASK) >> MIPS_PWSIZE_MDW_SHIFT,
2459 (config & MIPS_PWSIZE_PTW_MASK) >> MIPS_PWSIZE_PTW_SHIFT,
2460 (config & MIPS_PWSIZE_PTEW_MASK) >> MIPS_PWSIZE_PTEW_SHIFT);
2461
2462 pwctl = read_c0_pwctl();
2463 pr_debug("PWCtl (0x%x): PWEn: 0x%x XK: 0x%x XS: 0x%x XU: 0x%x DPH: 0x%x HugePg: 0x%x Psn: 0x%x\n",
2464 pwctl,
2465 (pwctl & MIPS_PWCTL_PWEN_MASK) >> MIPS_PWCTL_PWEN_SHIFT,
2466 (pwctl & MIPS_PWCTL_XK_MASK) >> MIPS_PWCTL_XK_SHIFT,
2467 (pwctl & MIPS_PWCTL_XS_MASK) >> MIPS_PWCTL_XS_SHIFT,
2468 (pwctl & MIPS_PWCTL_XU_MASK) >> MIPS_PWCTL_XU_SHIFT,
2469 (pwctl & MIPS_PWCTL_DPH_MASK) >> MIPS_PWCTL_DPH_SHIFT,
2470 (pwctl & MIPS_PWCTL_HUGEPG_MASK) >> MIPS_PWCTL_HUGEPG_SHIFT,
2471 (pwctl & MIPS_PWCTL_PSN_MASK) >> MIPS_PWCTL_PSN_SHIFT);
2472}
2473
2474static void config_htw_params(void)
2475{
2476 unsigned long pwfield, pwsize, ptei;
2477 unsigned int config;
2478
2479 /*
2480 * We are using 2-level page tables, so we only need to
2481 * setup GDW and PTW appropriately. UDW and MDW will remain 0.
2482 * The default value of GDI/UDI/MDI/PTI is 0xc. It is illegal to
2483 * write values less than 0xc in these fields because the entire
2484 * write will be dropped. As a result of which, we must preserve
2485 * the original reset values and overwrite only what we really want.
2486 */
2487
2488 pwfield = read_c0_pwfield();
2489 /* re-initialize the GDI field */
2490 pwfield &= ~MIPS_PWFIELD_GDI_MASK;
2491 pwfield |= PGDIR_SHIFT << MIPS_PWFIELD_GDI_SHIFT;
2492 /* re-initialize the PTI field including the even/odd bit */
2493 pwfield &= ~MIPS_PWFIELD_PTI_MASK;
2494 pwfield |= PAGE_SHIFT << MIPS_PWFIELD_PTI_SHIFT;
2495 if (CONFIG_PGTABLE_LEVELS >= 3) {
2496 pwfield &= ~MIPS_PWFIELD_MDI_MASK;
2497 pwfield |= PMD_SHIFT << MIPS_PWFIELD_MDI_SHIFT;
2498 }
2499 /* Set the PTEI right shift */
2500 ptei = _PAGE_GLOBAL_SHIFT << MIPS_PWFIELD_PTEI_SHIFT;
2501 pwfield |= ptei;
2502 write_c0_pwfield(pwfield);
2503 /* Check whether the PTEI value is supported */
2504 back_to_back_c0_hazard();
2505 pwfield = read_c0_pwfield();
2506 if (((pwfield & MIPS_PWFIELD_PTEI_MASK) << MIPS_PWFIELD_PTEI_SHIFT)
2507 != ptei) {
2508 pr_warn("Unsupported PTEI field value: 0x%lx. HTW will not be enabled",
2509 ptei);
2510 /*
2511 * Drop option to avoid HTW being enabled via another path
2512 * (eg htw_reset())
2513 */
2514 current_cpu_data.options &= ~MIPS_CPU_HTW;
2515 return;
2516 }
2517
2518 pwsize = ilog2(PTRS_PER_PGD) << MIPS_PWSIZE_GDW_SHIFT;
2519 pwsize |= ilog2(PTRS_PER_PTE) << MIPS_PWSIZE_PTW_SHIFT;
2520 if (CONFIG_PGTABLE_LEVELS >= 3)
2521 pwsize |= ilog2(PTRS_PER_PMD) << MIPS_PWSIZE_MDW_SHIFT;
2522
2523 /* Set pointer size to size of directory pointers */
2524 if (IS_ENABLED(CONFIG_64BIT))
2525 pwsize |= MIPS_PWSIZE_PS_MASK;
2526 /* PTEs may be multiple pointers long (e.g. with XPA) */
2527 pwsize |= ((PTE_T_LOG2 - PGD_T_LOG2) << MIPS_PWSIZE_PTEW_SHIFT)
2528 & MIPS_PWSIZE_PTEW_MASK;
2529
2530 write_c0_pwsize(pwsize);
2531
2532 /* Make sure everything is set before we enable the HTW */
2533 back_to_back_c0_hazard();
2534
2535 /*
2536 * Enable HTW (and only for XUSeg on 64-bit), and disable the rest of
2537 * the pwctl fields.
2538 */
2539 config = 1 << MIPS_PWCTL_PWEN_SHIFT;
2540 if (IS_ENABLED(CONFIG_64BIT))
2541 config |= MIPS_PWCTL_XU_MASK;
2542 write_c0_pwctl(config);
2543 pr_info("Hardware Page Table Walker enabled\n");
2544
2545 print_htw_config();
2546}
2547
2548static void config_xpa_params(void)
2549{
2550#ifdef CONFIG_XPA
2551 unsigned int pagegrain;
2552
2553 if (mips_xpa_disabled) {
2554 pr_info("Extended Physical Addressing (XPA) disabled\n");
2555 return;
2556 }
2557
2558 pagegrain = read_c0_pagegrain();
2559 write_c0_pagegrain(pagegrain | PG_ELPA);
2560 back_to_back_c0_hazard();
2561 pagegrain = read_c0_pagegrain();
2562
2563 if (pagegrain & PG_ELPA)
2564 pr_info("Extended Physical Addressing (XPA) enabled\n");
2565 else
2566 panic("Extended Physical Addressing (XPA) disabled");
2567#endif
2568}
2569
2570static void check_pabits(void)
2571{
2572 unsigned long entry;
2573 unsigned pabits, fillbits;
2574
2575 if (!cpu_has_rixi || _PAGE_NO_EXEC == 0) {
2576 /*
2577 * We'll only be making use of the fact that we can rotate bits
2578 * into the fill if the CPU supports RIXI, so don't bother
2579 * probing this for CPUs which don't.
2580 */
2581 return;
2582 }
2583
2584 write_c0_entrylo0(~0ul);
2585 back_to_back_c0_hazard();
2586 entry = read_c0_entrylo0();
2587
2588 /* clear all non-PFN bits */
2589 entry &= ~((1 << MIPS_ENTRYLO_PFN_SHIFT) - 1);
2590 entry &= ~(MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI);
2591
2592 /* find a lower bound on PABITS, and upper bound on fill bits */
2593 pabits = fls_long(entry) + 6;
2594 fillbits = max_t(int, (int)BITS_PER_LONG - pabits, 0);
2595
2596 /* minus the RI & XI bits */
2597 fillbits -= min_t(unsigned, fillbits, 2);
2598
2599 if (fillbits >= ilog2(_PAGE_NO_EXEC))
2600 fill_includes_sw_bits = true;
2601
2602 pr_debug("Entry* registers contain %u fill bits\n", fillbits);
2603}
2604
2605void build_tlb_refill_handler(void)
2606{
2607 /*
2608 * The refill handler is generated per-CPU, multi-node systems
2609 * may have local storage for it. The other handlers are only
2610 * needed once.
2611 */
2612 static int run_once = 0;
2613
2614 if (IS_ENABLED(CONFIG_XPA) && !cpu_has_rixi)
2615 panic("Kernels supporting XPA currently require CPUs with RIXI");
2616
2617 output_pgtable_bits_defines();
2618 check_pabits();
2619
2620#ifdef CONFIG_64BIT
2621 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
2622#endif
2623
2624 if (cpu_has_3kex) {
2625#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
2626 if (!run_once) {
2627 build_setup_pgd();
2628 build_r3000_tlb_refill_handler();
2629 build_r3000_tlb_load_handler();
2630 build_r3000_tlb_store_handler();
2631 build_r3000_tlb_modify_handler();
2632 flush_tlb_handlers();
2633 run_once++;
2634 }
2635#else
2636 panic("No R3000 TLB refill handler");
2637#endif
2638 return;
2639 }
2640
2641 if (cpu_has_ldpte)
2642 setup_pw();
2643
2644 if (!run_once) {
2645 scratch_reg = allocate_kscratch();
2646 build_setup_pgd();
2647 build_r4000_tlb_load_handler();
2648 build_r4000_tlb_store_handler();
2649 build_r4000_tlb_modify_handler();
2650 if (cpu_has_ldpte)
2651 build_loongson3_tlb_refill_handler();
2652 else
2653 build_r4000_tlb_refill_handler();
2654 flush_tlb_handlers();
2655 run_once++;
2656 }
2657 if (cpu_has_xpa)
2658 config_xpa_params();
2659 if (cpu_has_htw)
2660 config_htw_params();
2661}
diff --git a/arch/mips/mm/uasm-micromips.c b/arch/mips/mm/uasm-micromips.c
new file mode 100644
index 000000000..75ef90486
--- /dev/null
+++ b/arch/mips/mm/uasm-micromips.c
@@ -0,0 +1,232 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * A small micro-assembler. It is intentionally kept simple, does only
7 * support a subset of instructions, and does not try to hide pipeline
8 * effects like branch delay slots.
9 *
10 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
11 * Copyright (C) 2005, 2007 Maciej W. Rozycki
12 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
13 * Copyright (C) 2012, 2013 MIPS Technologies, Inc. All rights reserved.
14 */
15
16#include <linux/kernel.h>
17#include <linux/types.h>
18
19#include <asm/inst.h>
20#include <asm/elf.h>
21#include <asm/bugs.h>
22#include <asm/uasm.h>
23
24#define RS_MASK 0x1f
25#define RS_SH 16
26#define RT_MASK 0x1f
27#define RT_SH 21
28#define SCIMM_MASK 0x3ff
29#define SCIMM_SH 16
30
31/* This macro sets the non-variable bits of an instruction. */
32#define M(a, b, c, d, e, f) \
33 ((a) << OP_SH \
34 | (b) << RT_SH \
35 | (c) << RS_SH \
36 | (d) << RD_SH \
37 | (e) << RE_SH \
38 | (f) << FUNC_SH)
39
40#include "uasm.c"
41
42static const struct insn insn_table_MM[insn_invalid] = {
43 [insn_addu] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_addu32_op), RT | RS | RD},
44 [insn_addiu] = {M(mm_addiu32_op, 0, 0, 0, 0, 0), RT | RS | SIMM},
45 [insn_and] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_and_op), RT | RS | RD},
46 [insn_andi] = {M(mm_andi32_op, 0, 0, 0, 0, 0), RT | RS | UIMM},
47 [insn_beq] = {M(mm_beq32_op, 0, 0, 0, 0, 0), RS | RT | BIMM},
48 [insn_beql] = {0, 0},
49 [insn_bgez] = {M(mm_pool32i_op, mm_bgez_op, 0, 0, 0, 0), RS | BIMM},
50 [insn_bgezl] = {0, 0},
51 [insn_bltz] = {M(mm_pool32i_op, mm_bltz_op, 0, 0, 0, 0), RS | BIMM},
52 [insn_bltzl] = {0, 0},
53 [insn_bne] = {M(mm_bne32_op, 0, 0, 0, 0, 0), RT | RS | BIMM},
54 [insn_cache] = {M(mm_pool32b_op, 0, 0, mm_cache_func, 0, 0), RT | RS | SIMM},
55 [insn_cfc1] = {M(mm_pool32f_op, 0, 0, 0, mm_cfc1_op, mm_32f_73_op), RT | RS},
56 [insn_cfcmsa] = {M(mm_pool32s_op, 0, msa_cfc_op, 0, 0, mm_32s_elm_op), RD | RE},
57 [insn_ctc1] = {M(mm_pool32f_op, 0, 0, 0, mm_ctc1_op, mm_32f_73_op), RT | RS},
58 [insn_ctcmsa] = {M(mm_pool32s_op, 0, msa_ctc_op, 0, 0, mm_32s_elm_op), RD | RE},
59 [insn_daddu] = {0, 0},
60 [insn_daddiu] = {0, 0},
61 [insn_di] = {M(mm_pool32a_op, 0, 0, 0, mm_di_op, mm_pool32axf_op), RS},
62 [insn_divu] = {M(mm_pool32a_op, 0, 0, 0, mm_divu_op, mm_pool32axf_op), RT | RS},
63 [insn_dmfc0] = {0, 0},
64 [insn_dmtc0] = {0, 0},
65 [insn_dsll] = {0, 0},
66 [insn_dsll32] = {0, 0},
67 [insn_dsra] = {0, 0},
68 [insn_dsrl] = {0, 0},
69 [insn_dsrl32] = {0, 0},
70 [insn_drotr] = {0, 0},
71 [insn_drotr32] = {0, 0},
72 [insn_dsubu] = {0, 0},
73 [insn_eret] = {M(mm_pool32a_op, 0, 0, 0, mm_eret_op, mm_pool32axf_op), 0},
74 [insn_ins] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_ins_op), RT | RS | RD | RE},
75 [insn_ext] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_ext_op), RT | RS | RD | RE},
76 [insn_j] = {M(mm_j32_op, 0, 0, 0, 0, 0), JIMM},
77 [insn_jal] = {M(mm_jal32_op, 0, 0, 0, 0, 0), JIMM},
78 [insn_jalr] = {M(mm_pool32a_op, 0, 0, 0, mm_jalr_op, mm_pool32axf_op), RT | RS},
79 [insn_jr] = {M(mm_pool32a_op, 0, 0, 0, mm_jalr_op, mm_pool32axf_op), RS},
80 [insn_lb] = {M(mm_lb32_op, 0, 0, 0, 0, 0), RT | RS | SIMM},
81 [insn_ld] = {0, 0},
82 [insn_lh] = {M(mm_lh32_op, 0, 0, 0, 0, 0), RT | RS | SIMM},
83 [insn_ll] = {M(mm_pool32c_op, 0, 0, (mm_ll_func << 1), 0, 0), RS | RT | SIMM},
84 [insn_lld] = {0, 0},
85 [insn_lui] = {M(mm_pool32i_op, mm_lui_op, 0, 0, 0, 0), RS | SIMM},
86 [insn_lw] = {M(mm_lw32_op, 0, 0, 0, 0, 0), RT | RS | SIMM},
87 [insn_mfc0] = {M(mm_pool32a_op, 0, 0, 0, mm_mfc0_op, mm_pool32axf_op), RT | RS | RD},
88 [insn_mfhi] = {M(mm_pool32a_op, 0, 0, 0, mm_mfhi32_op, mm_pool32axf_op), RS},
89 [insn_mflo] = {M(mm_pool32a_op, 0, 0, 0, mm_mflo32_op, mm_pool32axf_op), RS},
90 [insn_mtc0] = {M(mm_pool32a_op, 0, 0, 0, mm_mtc0_op, mm_pool32axf_op), RT | RS | RD},
91 [insn_mthi] = {M(mm_pool32a_op, 0, 0, 0, mm_mthi32_op, mm_pool32axf_op), RS},
92 [insn_mtlo] = {M(mm_pool32a_op, 0, 0, 0, mm_mtlo32_op, mm_pool32axf_op), RS},
93 [insn_mul] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_mul_op), RT | RS | RD},
94 [insn_or] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_or32_op), RT | RS | RD},
95 [insn_ori] = {M(mm_ori32_op, 0, 0, 0, 0, 0), RT | RS | UIMM},
96 [insn_pref] = {M(mm_pool32c_op, 0, 0, (mm_pref_func << 1), 0, 0), RT | RS | SIMM},
97 [insn_rfe] = {0, 0},
98 [insn_sc] = {M(mm_pool32c_op, 0, 0, (mm_sc_func << 1), 0, 0), RT | RS | SIMM},
99 [insn_scd] = {0, 0},
100 [insn_sd] = {0, 0},
101 [insn_sll] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_sll32_op), RT | RS | RD},
102 [insn_sllv] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_sllv32_op), RT | RS | RD},
103 [insn_slt] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_slt_op), RT | RS | RD},
104 [insn_sltiu] = {M(mm_sltiu32_op, 0, 0, 0, 0, 0), RT | RS | SIMM},
105 [insn_sltu] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_sltu_op), RT | RS | RD},
106 [insn_sra] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_sra_op), RT | RS | RD},
107 [insn_srav] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_srav_op), RT | RS | RD},
108 [insn_srl] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_srl32_op), RT | RS | RD},
109 [insn_srlv] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_srlv32_op), RT | RS | RD},
110 [insn_rotr] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_rotr_op), RT | RS | RD},
111 [insn_subu] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_subu32_op), RT | RS | RD},
112 [insn_sw] = {M(mm_sw32_op, 0, 0, 0, 0, 0), RT | RS | SIMM},
113 [insn_sync] = {M(mm_pool32a_op, 0, 0, 0, mm_sync_op, mm_pool32axf_op), RS},
114 [insn_tlbp] = {M(mm_pool32a_op, 0, 0, 0, mm_tlbp_op, mm_pool32axf_op), 0},
115 [insn_tlbr] = {M(mm_pool32a_op, 0, 0, 0, mm_tlbr_op, mm_pool32axf_op), 0},
116 [insn_tlbwi] = {M(mm_pool32a_op, 0, 0, 0, mm_tlbwi_op, mm_pool32axf_op), 0},
117 [insn_tlbwr] = {M(mm_pool32a_op, 0, 0, 0, mm_tlbwr_op, mm_pool32axf_op), 0},
118 [insn_wait] = {M(mm_pool32a_op, 0, 0, 0, mm_wait_op, mm_pool32axf_op), SCIMM},
119 [insn_wsbh] = {M(mm_pool32a_op, 0, 0, 0, mm_wsbh_op, mm_pool32axf_op), RT | RS},
120 [insn_xor] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_xor32_op), RT | RS | RD},
121 [insn_xori] = {M(mm_xori32_op, 0, 0, 0, 0, 0), RT | RS | UIMM},
122 [insn_dins] = {0, 0},
123 [insn_dinsm] = {0, 0},
124 [insn_syscall] = {M(mm_pool32a_op, 0, 0, 0, mm_syscall_op, mm_pool32axf_op), SCIMM},
125 [insn_bbit0] = {0, 0},
126 [insn_bbit1] = {0, 0},
127 [insn_lwx] = {0, 0},
128 [insn_ldx] = {0, 0},
129};
130
131#undef M
132
133static inline u32 build_bimm(s32 arg)
134{
135 WARN(arg > 0xffff || arg < -0x10000,
136 KERN_WARNING "Micro-assembler field overflow\n");
137
138 WARN(arg & 0x3, KERN_WARNING "Invalid micro-assembler branch target\n");
139
140 return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 1) & 0x7fff);
141}
142
143static inline u32 build_jimm(u32 arg)
144{
145
146 WARN(arg & ~((JIMM_MASK << 2) | 1),
147 KERN_WARNING "Micro-assembler field overflow\n");
148
149 return (arg >> 1) & JIMM_MASK;
150}
151
152/*
153 * The order of opcode arguments is implicitly left to right,
154 * starting with RS and ending with FUNC or IMM.
155 */
156static void build_insn(u32 **buf, enum opcode opc, ...)
157{
158 const struct insn *ip;
159 va_list ap;
160 u32 op;
161
162 if (opc < 0 || opc >= insn_invalid ||
163 (opc == insn_daddiu && r4k_daddiu_bug()) ||
164 (insn_table_MM[opc].match == 0 && insn_table_MM[opc].fields == 0))
165 panic("Unsupported Micro-assembler instruction %d", opc);
166
167 ip = &insn_table_MM[opc];
168
169 op = ip->match;
170 va_start(ap, opc);
171 if (ip->fields & RS) {
172 if (opc == insn_mfc0 || opc == insn_mtc0 ||
173 opc == insn_cfc1 || opc == insn_ctc1)
174 op |= build_rt(va_arg(ap, u32));
175 else
176 op |= build_rs(va_arg(ap, u32));
177 }
178 if (ip->fields & RT) {
179 if (opc == insn_mfc0 || opc == insn_mtc0 ||
180 opc == insn_cfc1 || opc == insn_ctc1)
181 op |= build_rs(va_arg(ap, u32));
182 else
183 op |= build_rt(va_arg(ap, u32));
184 }
185 if (ip->fields & RD)
186 op |= build_rd(va_arg(ap, u32));
187 if (ip->fields & RE)
188 op |= build_re(va_arg(ap, u32));
189 if (ip->fields & SIMM)
190 op |= build_simm(va_arg(ap, s32));
191 if (ip->fields & UIMM)
192 op |= build_uimm(va_arg(ap, u32));
193 if (ip->fields & BIMM)
194 op |= build_bimm(va_arg(ap, s32));
195 if (ip->fields & JIMM)
196 op |= build_jimm(va_arg(ap, u32));
197 if (ip->fields & FUNC)
198 op |= build_func(va_arg(ap, u32));
199 if (ip->fields & SET)
200 op |= build_set(va_arg(ap, u32));
201 if (ip->fields & SCIMM)
202 op |= build_scimm(va_arg(ap, u32));
203 va_end(ap);
204
205#ifdef CONFIG_CPU_LITTLE_ENDIAN
206 **buf = ((op & 0xffff) << 16) | (op >> 16);
207#else
208 **buf = op;
209#endif
210 (*buf)++;
211}
212
213static inline void
214__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab)
215{
216 long laddr = (long)lab->addr;
217 long raddr = (long)rel->addr;
218
219 switch (rel->type) {
220 case R_MIPS_PC16:
221#ifdef CONFIG_CPU_LITTLE_ENDIAN
222 *rel->addr |= (build_bimm(laddr - (raddr + 4)) << 16);
223#else
224 *rel->addr |= build_bimm(laddr - (raddr + 4));
225#endif
226 break;
227
228 default:
229 panic("Unsupported Micro-assembler relocation %d",
230 rel->type);
231 }
232}
diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c
new file mode 100644
index 000000000..7154a1d99
--- /dev/null
+++ b/arch/mips/mm/uasm-mips.c
@@ -0,0 +1,290 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * A small micro-assembler. It is intentionally kept simple, does only
7 * support a subset of instructions, and does not try to hide pipeline
8 * effects like branch delay slots.
9 *
10 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
11 * Copyright (C) 2005, 2007 Maciej W. Rozycki
12 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
13 * Copyright (C) 2012, 2013 MIPS Technologies, Inc. All rights reserved.
14 */
15
16#include <linux/kernel.h>
17#include <linux/types.h>
18
19#include <asm/inst.h>
20#include <asm/elf.h>
21#include <asm/bugs.h>
22#include <asm/uasm.h>
23
24#define RS_MASK 0x1f
25#define RS_SH 21
26#define RT_MASK 0x1f
27#define RT_SH 16
28#define SCIMM_MASK 0xfffff
29#define SCIMM_SH 6
30
31/* This macro sets the non-variable bits of an instruction. */
32#define M(a, b, c, d, e, f) \
33 ((a) << OP_SH \
34 | (b) << RS_SH \
35 | (c) << RT_SH \
36 | (d) << RD_SH \
37 | (e) << RE_SH \
38 | (f) << FUNC_SH)
39
40/* This macro sets the non-variable bits of an R6 instruction. */
41#define M6(a, b, c, d, e) \
42 ((a) << OP_SH \
43 | (b) << RS_SH \
44 | (c) << RT_SH \
45 | (d) << SIMM9_SH \
46 | (e) << FUNC_SH)
47
48#include "uasm.c"
49
50static const struct insn insn_table[insn_invalid] = {
51 [insn_addiu] = {M(addiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
52 [insn_addu] = {M(spec_op, 0, 0, 0, 0, addu_op), RS | RT | RD},
53 [insn_and] = {M(spec_op, 0, 0, 0, 0, and_op), RS | RT | RD},
54 [insn_andi] = {M(andi_op, 0, 0, 0, 0, 0), RS | RT | UIMM},
55 [insn_bbit0] = {M(lwc2_op, 0, 0, 0, 0, 0), RS | RT | BIMM},
56 [insn_bbit1] = {M(swc2_op, 0, 0, 0, 0, 0), RS | RT | BIMM},
57 [insn_beq] = {M(beq_op, 0, 0, 0, 0, 0), RS | RT | BIMM},
58 [insn_beql] = {M(beql_op, 0, 0, 0, 0, 0), RS | RT | BIMM},
59 [insn_bgez] = {M(bcond_op, 0, bgez_op, 0, 0, 0), RS | BIMM},
60 [insn_bgezl] = {M(bcond_op, 0, bgezl_op, 0, 0, 0), RS | BIMM},
61 [insn_bgtz] = {M(bgtz_op, 0, 0, 0, 0, 0), RS | BIMM},
62 [insn_blez] = {M(blez_op, 0, 0, 0, 0, 0), RS | BIMM},
63 [insn_bltz] = {M(bcond_op, 0, bltz_op, 0, 0, 0), RS | BIMM},
64 [insn_bltzl] = {M(bcond_op, 0, bltzl_op, 0, 0, 0), RS | BIMM},
65 [insn_bne] = {M(bne_op, 0, 0, 0, 0, 0), RS | RT | BIMM},
66 [insn_break] = {M(spec_op, 0, 0, 0, 0, break_op), SCIMM},
67#ifndef CONFIG_CPU_MIPSR6
68 [insn_cache] = {M(cache_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
69#else
70 [insn_cache] = {M6(spec3_op, 0, 0, 0, cache6_op), RS | RT | SIMM9},
71#endif
72 [insn_cfc1] = {M(cop1_op, cfc_op, 0, 0, 0, 0), RT | RD},
73 [insn_cfcmsa] = {M(msa_op, 0, msa_cfc_op, 0, 0, msa_elm_op), RD | RE},
74 [insn_ctc1] = {M(cop1_op, ctc_op, 0, 0, 0, 0), RT | RD},
75 [insn_ctcmsa] = {M(msa_op, 0, msa_ctc_op, 0, 0, msa_elm_op), RD | RE},
76 [insn_daddiu] = {M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
77 [insn_daddu] = {M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD},
78 [insn_ddivu] = {M(spec_op, 0, 0, 0, 0, ddivu_op), RS | RT},
79 [insn_ddivu_r6] = {M(spec_op, 0, 0, 0, ddivu_ddivu6_op, ddivu_op),
80 RS | RT | RD},
81 [insn_di] = {M(cop0_op, mfmc0_op, 0, 12, 0, 0), RT},
82 [insn_dins] = {M(spec3_op, 0, 0, 0, 0, dins_op), RS | RT | RD | RE},
83 [insn_dinsm] = {M(spec3_op, 0, 0, 0, 0, dinsm_op), RS | RT | RD | RE},
84 [insn_dinsu] = {M(spec3_op, 0, 0, 0, 0, dinsu_op), RS | RT | RD | RE},
85 [insn_divu] = {M(spec_op, 0, 0, 0, 0, divu_op), RS | RT},
86 [insn_divu_r6] = {M(spec_op, 0, 0, 0, divu_divu6_op, divu_op),
87 RS | RT | RD},
88 [insn_dmfc0] = {M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET},
89 [insn_dmodu] = {M(spec_op, 0, 0, 0, ddivu_dmodu_op, ddivu_op),
90 RS | RT | RD},
91 [insn_dmtc0] = {M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET},
92 [insn_dmultu] = {M(spec_op, 0, 0, 0, 0, dmultu_op), RS | RT},
93 [insn_dmulu] = {M(spec_op, 0, 0, 0, dmult_dmul_op, dmultu_op),
94 RS | RT | RD},
95 [insn_drotr] = {M(spec_op, 1, 0, 0, 0, dsrl_op), RT | RD | RE},
96 [insn_drotr32] = {M(spec_op, 1, 0, 0, 0, dsrl32_op), RT | RD | RE},
97 [insn_dsbh] = {M(spec3_op, 0, 0, 0, dsbh_op, dbshfl_op), RT | RD},
98 [insn_dshd] = {M(spec3_op, 0, 0, 0, dshd_op, dbshfl_op), RT | RD},
99 [insn_dsll] = {M(spec_op, 0, 0, 0, 0, dsll_op), RT | RD | RE},
100 [insn_dsll32] = {M(spec_op, 0, 0, 0, 0, dsll32_op), RT | RD | RE},
101 [insn_dsllv] = {M(spec_op, 0, 0, 0, 0, dsllv_op), RS | RT | RD},
102 [insn_dsra] = {M(spec_op, 0, 0, 0, 0, dsra_op), RT | RD | RE},
103 [insn_dsra32] = {M(spec_op, 0, 0, 0, 0, dsra32_op), RT | RD | RE},
104 [insn_dsrav] = {M(spec_op, 0, 0, 0, 0, dsrav_op), RS | RT | RD},
105 [insn_dsrl] = {M(spec_op, 0, 0, 0, 0, dsrl_op), RT | RD | RE},
106 [insn_dsrl32] = {M(spec_op, 0, 0, 0, 0, dsrl32_op), RT | RD | RE},
107 [insn_dsrlv] = {M(spec_op, 0, 0, 0, 0, dsrlv_op), RS | RT | RD},
108 [insn_dsubu] = {M(spec_op, 0, 0, 0, 0, dsubu_op), RS | RT | RD},
109 [insn_eret] = {M(cop0_op, cop_op, 0, 0, 0, eret_op), 0},
110 [insn_ext] = {M(spec3_op, 0, 0, 0, 0, ext_op), RS | RT | RD | RE},
111 [insn_ins] = {M(spec3_op, 0, 0, 0, 0, ins_op), RS | RT | RD | RE},
112 [insn_j] = {M(j_op, 0, 0, 0, 0, 0), JIMM},
113 [insn_jal] = {M(jal_op, 0, 0, 0, 0, 0), JIMM},
114 [insn_jalr] = {M(spec_op, 0, 0, 0, 0, jalr_op), RS | RD},
115#ifndef CONFIG_CPU_MIPSR6
116 [insn_jr] = {M(spec_op, 0, 0, 0, 0, jr_op), RS},
117#else
118 [insn_jr] = {M(spec_op, 0, 0, 0, 0, jalr_op), RS},
119#endif
120 [insn_lb] = {M(lb_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
121 [insn_lbu] = {M(lbu_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
122 [insn_ld] = {M(ld_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
123 [insn_lddir] = {M(lwc2_op, 0, 0, 0, lddir_op, mult_op), RS | RT | RD},
124 [insn_ldpte] = {M(lwc2_op, 0, 0, 0, ldpte_op, mult_op), RS | RD},
125 [insn_ldx] = {M(spec3_op, 0, 0, 0, ldx_op, lx_op), RS | RT | RD},
126 [insn_lh] = {M(lh_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
127 [insn_lhu] = {M(lhu_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
128#ifndef CONFIG_CPU_MIPSR6
129 [insn_ll] = {M(ll_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
130 [insn_lld] = {M(lld_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
131#else
132 [insn_ll] = {M6(spec3_op, 0, 0, 0, ll6_op), RS | RT | SIMM9},
133 [insn_lld] = {M6(spec3_op, 0, 0, 0, lld6_op), RS | RT | SIMM9},
134#endif
135 [insn_lui] = {M(lui_op, 0, 0, 0, 0, 0), RT | SIMM},
136 [insn_lw] = {M(lw_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
137 [insn_lwu] = {M(lwu_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
138 [insn_lwx] = {M(spec3_op, 0, 0, 0, lwx_op, lx_op), RS | RT | RD},
139 [insn_mfc0] = {M(cop0_op, mfc_op, 0, 0, 0, 0), RT | RD | SET},
140 [insn_mfhc0] = {M(cop0_op, mfhc0_op, 0, 0, 0, 0), RT | RD | SET},
141 [insn_mfhi] = {M(spec_op, 0, 0, 0, 0, mfhi_op), RD},
142 [insn_mflo] = {M(spec_op, 0, 0, 0, 0, mflo_op), RD},
143 [insn_modu] = {M(spec_op, 0, 0, 0, divu_modu_op, divu_op),
144 RS | RT | RD},
145 [insn_movn] = {M(spec_op, 0, 0, 0, 0, movn_op), RS | RT | RD},
146 [insn_movz] = {M(spec_op, 0, 0, 0, 0, movz_op), RS | RT | RD},
147 [insn_mtc0] = {M(cop0_op, mtc_op, 0, 0, 0, 0), RT | RD | SET},
148 [insn_mthc0] = {M(cop0_op, mthc0_op, 0, 0, 0, 0), RT | RD | SET},
149 [insn_mthi] = {M(spec_op, 0, 0, 0, 0, mthi_op), RS},
150 [insn_mtlo] = {M(spec_op, 0, 0, 0, 0, mtlo_op), RS},
151 [insn_mulu] = {M(spec_op, 0, 0, 0, multu_mulu_op, multu_op),
152 RS | RT | RD},
153#ifndef CONFIG_CPU_MIPSR6
154 [insn_mul] = {M(spec2_op, 0, 0, 0, 0, mul_op), RS | RT | RD},
155#else
156 [insn_mul] = {M(spec_op, 0, 0, 0, mult_mul_op, mult_op), RS | RT | RD},
157#endif
158 [insn_multu] = {M(spec_op, 0, 0, 0, 0, multu_op), RS | RT},
159 [insn_nor] = {M(spec_op, 0, 0, 0, 0, nor_op), RS | RT | RD},
160 [insn_or] = {M(spec_op, 0, 0, 0, 0, or_op), RS | RT | RD},
161 [insn_ori] = {M(ori_op, 0, 0, 0, 0, 0), RS | RT | UIMM},
162#ifndef CONFIG_CPU_MIPSR6
163 [insn_pref] = {M(pref_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
164#else
165 [insn_pref] = {M6(spec3_op, 0, 0, 0, pref6_op), RS | RT | SIMM9},
166#endif
167 [insn_rfe] = {M(cop0_op, cop_op, 0, 0, 0, rfe_op), 0},
168 [insn_rotr] = {M(spec_op, 1, 0, 0, 0, srl_op), RT | RD | RE},
169 [insn_sb] = {M(sb_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
170#ifndef CONFIG_CPU_MIPSR6
171 [insn_sc] = {M(sc_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
172 [insn_scd] = {M(scd_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
173#else
174 [insn_sc] = {M6(spec3_op, 0, 0, 0, sc6_op), RS | RT | SIMM9},
175 [insn_scd] = {M6(spec3_op, 0, 0, 0, scd6_op), RS | RT | SIMM9},
176#endif
177 [insn_sd] = {M(sd_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
178 [insn_seleqz] = {M(spec_op, 0, 0, 0, 0, seleqz_op), RS | RT | RD},
179 [insn_selnez] = {M(spec_op, 0, 0, 0, 0, selnez_op), RS | RT | RD},
180 [insn_sh] = {M(sh_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
181 [insn_sll] = {M(spec_op, 0, 0, 0, 0, sll_op), RT | RD | RE},
182 [insn_sllv] = {M(spec_op, 0, 0, 0, 0, sllv_op), RS | RT | RD},
183 [insn_slt] = {M(spec_op, 0, 0, 0, 0, slt_op), RS | RT | RD},
184 [insn_slti] = {M(slti_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
185 [insn_sltiu] = {M(sltiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
186 [insn_sltu] = {M(spec_op, 0, 0, 0, 0, sltu_op), RS | RT | RD},
187 [insn_sra] = {M(spec_op, 0, 0, 0, 0, sra_op), RT | RD | RE},
188 [insn_srav] = {M(spec_op, 0, 0, 0, 0, srav_op), RS | RT | RD},
189 [insn_srl] = {M(spec_op, 0, 0, 0, 0, srl_op), RT | RD | RE},
190 [insn_srlv] = {M(spec_op, 0, 0, 0, 0, srlv_op), RS | RT | RD},
191 [insn_subu] = {M(spec_op, 0, 0, 0, 0, subu_op), RS | RT | RD},
192 [insn_sw] = {M(sw_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
193 [insn_sync] = {M(spec_op, 0, 0, 0, 0, sync_op), RE},
194 [insn_syscall] = {M(spec_op, 0, 0, 0, 0, syscall_op), SCIMM},
195 [insn_tlbp] = {M(cop0_op, cop_op, 0, 0, 0, tlbp_op), 0},
196 [insn_tlbr] = {M(cop0_op, cop_op, 0, 0, 0, tlbr_op), 0},
197 [insn_tlbwi] = {M(cop0_op, cop_op, 0, 0, 0, tlbwi_op), 0},
198 [insn_tlbwr] = {M(cop0_op, cop_op, 0, 0, 0, tlbwr_op), 0},
199 [insn_wait] = {M(cop0_op, cop_op, 0, 0, 0, wait_op), SCIMM},
200 [insn_wsbh] = {M(spec3_op, 0, 0, 0, wsbh_op, bshfl_op), RT | RD},
201 [insn_xor] = {M(spec_op, 0, 0, 0, 0, xor_op), RS | RT | RD},
202 [insn_xori] = {M(xori_op, 0, 0, 0, 0, 0), RS | RT | UIMM},
203 [insn_yield] = {M(spec3_op, 0, 0, 0, 0, yield_op), RS | RD},
204};
205
206#undef M
207
208static inline u32 build_bimm(s32 arg)
209{
210 WARN(arg > 0x1ffff || arg < -0x20000,
211 KERN_WARNING "Micro-assembler field overflow\n");
212
213 WARN(arg & 0x3, KERN_WARNING "Invalid micro-assembler branch target\n");
214
215 return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff);
216}
217
218static inline u32 build_jimm(u32 arg)
219{
220 WARN(arg & ~(JIMM_MASK << 2),
221 KERN_WARNING "Micro-assembler field overflow\n");
222
223 return (arg >> 2) & JIMM_MASK;
224}
225
226/*
227 * The order of opcode arguments is implicitly left to right,
228 * starting with RS and ending with FUNC or IMM.
229 */
230static void build_insn(u32 **buf, enum opcode opc, ...)
231{
232 const struct insn *ip;
233 va_list ap;
234 u32 op;
235
236 if (opc < 0 || opc >= insn_invalid ||
237 (opc == insn_daddiu && r4k_daddiu_bug()) ||
238 (insn_table[opc].match == 0 && insn_table[opc].fields == 0))
239 panic("Unsupported Micro-assembler instruction %d", opc);
240
241 ip = &insn_table[opc];
242
243 op = ip->match;
244 va_start(ap, opc);
245 if (ip->fields & RS)
246 op |= build_rs(va_arg(ap, u32));
247 if (ip->fields & RT)
248 op |= build_rt(va_arg(ap, u32));
249 if (ip->fields & RD)
250 op |= build_rd(va_arg(ap, u32));
251 if (ip->fields & RE)
252 op |= build_re(va_arg(ap, u32));
253 if (ip->fields & SIMM)
254 op |= build_simm(va_arg(ap, s32));
255 if (ip->fields & UIMM)
256 op |= build_uimm(va_arg(ap, u32));
257 if (ip->fields & BIMM)
258 op |= build_bimm(va_arg(ap, s32));
259 if (ip->fields & JIMM)
260 op |= build_jimm(va_arg(ap, u32));
261 if (ip->fields & FUNC)
262 op |= build_func(va_arg(ap, u32));
263 if (ip->fields & SET)
264 op |= build_set(va_arg(ap, u32));
265 if (ip->fields & SCIMM)
266 op |= build_scimm(va_arg(ap, u32));
267 if (ip->fields & SIMM9)
268 op |= build_scimm9(va_arg(ap, u32));
269 va_end(ap);
270
271 **buf = op;
272 (*buf)++;
273}
274
275static inline void
276__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab)
277{
278 long laddr = (long)lab->addr;
279 long raddr = (long)rel->addr;
280
281 switch (rel->type) {
282 case R_MIPS_PC16:
283 *rel->addr |= build_bimm(laddr - (raddr + 4));
284 break;
285
286 default:
287 panic("Unsupported Micro-assembler relocation %d",
288 rel->type);
289 }
290}
diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c
new file mode 100644
index 000000000..81dd226d6
--- /dev/null
+++ b/arch/mips/mm/uasm.c
@@ -0,0 +1,643 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * A small micro-assembler. It is intentionally kept simple, does only
7 * support a subset of instructions, and does not try to hide pipeline
8 * effects like branch delay slots.
9 *
10 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
11 * Copyright (C) 2005, 2007 Maciej W. Rozycki
12 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
13 * Copyright (C) 2012, 2013 MIPS Technologies, Inc. All rights reserved.
14 */
15
16enum fields {
17 RS = 0x001,
18 RT = 0x002,
19 RD = 0x004,
20 RE = 0x008,
21 SIMM = 0x010,
22 UIMM = 0x020,
23 BIMM = 0x040,
24 JIMM = 0x080,
25 FUNC = 0x100,
26 SET = 0x200,
27 SCIMM = 0x400,
28 SIMM9 = 0x800,
29};
30
31#define OP_MASK 0x3f
32#define OP_SH 26
33#define RD_MASK 0x1f
34#define RD_SH 11
35#define RE_MASK 0x1f
36#define RE_SH 6
37#define IMM_MASK 0xffff
38#define IMM_SH 0
39#define JIMM_MASK 0x3ffffff
40#define JIMM_SH 0
41#define FUNC_MASK 0x3f
42#define FUNC_SH 0
43#define SET_MASK 0x7
44#define SET_SH 0
45#define SIMM9_SH 7
46#define SIMM9_MASK 0x1ff
47
48enum opcode {
49 insn_addiu, insn_addu, insn_and, insn_andi, insn_bbit0, insn_bbit1,
50 insn_beq, insn_beql, insn_bgez, insn_bgezl, insn_bgtz, insn_blez,
51 insn_bltz, insn_bltzl, insn_bne, insn_break, insn_cache, insn_cfc1,
52 insn_cfcmsa, insn_ctc1, insn_ctcmsa, insn_daddiu, insn_daddu, insn_ddivu,
53 insn_ddivu_r6, insn_di, insn_dins, insn_dinsm, insn_dinsu, insn_divu,
54 insn_divu_r6, insn_dmfc0, insn_dmodu, insn_dmtc0, insn_dmultu,
55 insn_dmulu, insn_drotr, insn_drotr32, insn_dsbh, insn_dshd, insn_dsll,
56 insn_dsll32, insn_dsllv, insn_dsra, insn_dsra32, insn_dsrav, insn_dsrl,
57 insn_dsrl32, insn_dsrlv, insn_dsubu, insn_eret, insn_ext, insn_ins,
58 insn_j, insn_jal, insn_jalr, insn_jr, insn_lb, insn_lbu, insn_ld,
59 insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu, insn_ll, insn_lld,
60 insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0, insn_mfhc0, insn_mfhi,
61 insn_mflo, insn_modu, insn_movn, insn_movz, insn_mtc0, insn_mthc0,
62 insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_nor,
63 insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc,
64 insn_scd, insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll,
65 insn_sllv, insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra,
66 insn_srav, insn_srl, insn_srlv, insn_subu, insn_sw, insn_sync,
67 insn_syscall, insn_tlbp, insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait,
68 insn_wsbh, insn_xor, insn_xori, insn_yield,
69 insn_invalid /* insn_invalid must be last */
70};
71
72struct insn {
73 u32 match;
74 enum fields fields;
75};
76
77static inline u32 build_rs(u32 arg)
78{
79 WARN(arg & ~RS_MASK, KERN_WARNING "Micro-assembler field overflow\n");
80
81 return (arg & RS_MASK) << RS_SH;
82}
83
84static inline u32 build_rt(u32 arg)
85{
86 WARN(arg & ~RT_MASK, KERN_WARNING "Micro-assembler field overflow\n");
87
88 return (arg & RT_MASK) << RT_SH;
89}
90
91static inline u32 build_rd(u32 arg)
92{
93 WARN(arg & ~RD_MASK, KERN_WARNING "Micro-assembler field overflow\n");
94
95 return (arg & RD_MASK) << RD_SH;
96}
97
98static inline u32 build_re(u32 arg)
99{
100 WARN(arg & ~RE_MASK, KERN_WARNING "Micro-assembler field overflow\n");
101
102 return (arg & RE_MASK) << RE_SH;
103}
104
105static inline u32 build_simm(s32 arg)
106{
107 WARN(arg > 0x7fff || arg < -0x8000,
108 KERN_WARNING "Micro-assembler field overflow\n");
109
110 return arg & 0xffff;
111}
112
113static inline u32 build_uimm(u32 arg)
114{
115 WARN(arg & ~IMM_MASK, KERN_WARNING "Micro-assembler field overflow\n");
116
117 return arg & IMM_MASK;
118}
119
120static inline u32 build_scimm(u32 arg)
121{
122 WARN(arg & ~SCIMM_MASK,
123 KERN_WARNING "Micro-assembler field overflow\n");
124
125 return (arg & SCIMM_MASK) << SCIMM_SH;
126}
127
128static inline u32 build_scimm9(s32 arg)
129{
130 WARN((arg > 0xff || arg < -0x100),
131 KERN_WARNING "Micro-assembler field overflow\n");
132
133 return (arg & SIMM9_MASK) << SIMM9_SH;
134}
135
136static inline u32 build_func(u32 arg)
137{
138 WARN(arg & ~FUNC_MASK, KERN_WARNING "Micro-assembler field overflow\n");
139
140 return arg & FUNC_MASK;
141}
142
143static inline u32 build_set(u32 arg)
144{
145 WARN(arg & ~SET_MASK, KERN_WARNING "Micro-assembler field overflow\n");
146
147 return arg & SET_MASK;
148}
149
150static void build_insn(u32 **buf, enum opcode opc, ...);
151
152#define I_u1u2u3(op) \
153Ip_u1u2u3(op) \
154{ \
155 build_insn(buf, insn##op, a, b, c); \
156} \
157UASM_EXPORT_SYMBOL(uasm_i##op);
158
159#define I_s3s1s2(op) \
160Ip_s3s1s2(op) \
161{ \
162 build_insn(buf, insn##op, b, c, a); \
163} \
164UASM_EXPORT_SYMBOL(uasm_i##op);
165
166#define I_u2u1u3(op) \
167Ip_u2u1u3(op) \
168{ \
169 build_insn(buf, insn##op, b, a, c); \
170} \
171UASM_EXPORT_SYMBOL(uasm_i##op);
172
173#define I_u3u2u1(op) \
174Ip_u3u2u1(op) \
175{ \
176 build_insn(buf, insn##op, c, b, a); \
177} \
178UASM_EXPORT_SYMBOL(uasm_i##op);
179
180#define I_u3u1u2(op) \
181Ip_u3u1u2(op) \
182{ \
183 build_insn(buf, insn##op, b, c, a); \
184} \
185UASM_EXPORT_SYMBOL(uasm_i##op);
186
187#define I_u1u2s3(op) \
188Ip_u1u2s3(op) \
189{ \
190 build_insn(buf, insn##op, a, b, c); \
191} \
192UASM_EXPORT_SYMBOL(uasm_i##op);
193
194#define I_u2s3u1(op) \
195Ip_u2s3u1(op) \
196{ \
197 build_insn(buf, insn##op, c, a, b); \
198} \
199UASM_EXPORT_SYMBOL(uasm_i##op);
200
201#define I_u2u1s3(op) \
202Ip_u2u1s3(op) \
203{ \
204 build_insn(buf, insn##op, b, a, c); \
205} \
206UASM_EXPORT_SYMBOL(uasm_i##op);
207
208#define I_u2u1msbu3(op) \
209Ip_u2u1msbu3(op) \
210{ \
211 build_insn(buf, insn##op, b, a, c+d-1, c); \
212} \
213UASM_EXPORT_SYMBOL(uasm_i##op);
214
215#define I_u2u1msb32u3(op) \
216Ip_u2u1msbu3(op) \
217{ \
218 build_insn(buf, insn##op, b, a, c+d-33, c); \
219} \
220UASM_EXPORT_SYMBOL(uasm_i##op);
221
222#define I_u2u1msb32msb3(op) \
223Ip_u2u1msbu3(op) \
224{ \
225 build_insn(buf, insn##op, b, a, c+d-33, c-32); \
226} \
227UASM_EXPORT_SYMBOL(uasm_i##op);
228
229#define I_u2u1msbdu3(op) \
230Ip_u2u1msbu3(op) \
231{ \
232 build_insn(buf, insn##op, b, a, d-1, c); \
233} \
234UASM_EXPORT_SYMBOL(uasm_i##op);
235
236#define I_u1u2(op) \
237Ip_u1u2(op) \
238{ \
239 build_insn(buf, insn##op, a, b); \
240} \
241UASM_EXPORT_SYMBOL(uasm_i##op);
242
243#define I_u2u1(op) \
244Ip_u1u2(op) \
245{ \
246 build_insn(buf, insn##op, b, a); \
247} \
248UASM_EXPORT_SYMBOL(uasm_i##op);
249
250#define I_u1s2(op) \
251Ip_u1s2(op) \
252{ \
253 build_insn(buf, insn##op, a, b); \
254} \
255UASM_EXPORT_SYMBOL(uasm_i##op);
256
257#define I_u1(op) \
258Ip_u1(op) \
259{ \
260 build_insn(buf, insn##op, a); \
261} \
262UASM_EXPORT_SYMBOL(uasm_i##op);
263
264#define I_0(op) \
265Ip_0(op) \
266{ \
267 build_insn(buf, insn##op); \
268} \
269UASM_EXPORT_SYMBOL(uasm_i##op);
270
271I_u2u1s3(_addiu)
272I_u3u1u2(_addu)
273I_u2u1u3(_andi)
274I_u3u1u2(_and)
275I_u1u2s3(_beq)
276I_u1u2s3(_beql)
277I_u1s2(_bgez)
278I_u1s2(_bgezl)
279I_u1s2(_bgtz)
280I_u1s2(_blez)
281I_u1s2(_bltz)
282I_u1s2(_bltzl)
283I_u1u2s3(_bne)
284I_u1(_break)
285I_u2s3u1(_cache)
286I_u1u2(_cfc1)
287I_u2u1(_cfcmsa)
288I_u1u2(_ctc1)
289I_u2u1(_ctcmsa)
290I_u1u2(_ddivu)
291I_u3u1u2(_ddivu_r6)
292I_u1u2u3(_dmfc0)
293I_u3u1u2(_dmodu)
294I_u1u2u3(_dmtc0)
295I_u1u2(_dmultu)
296I_u3u1u2(_dmulu)
297I_u2u1s3(_daddiu)
298I_u3u1u2(_daddu)
299I_u1(_di);
300I_u1u2(_divu)
301I_u3u1u2(_divu_r6)
302I_u2u1(_dsbh);
303I_u2u1(_dshd);
304I_u2u1u3(_dsll)
305I_u2u1u3(_dsll32)
306I_u3u2u1(_dsllv)
307I_u2u1u3(_dsra)
308I_u2u1u3(_dsra32)
309I_u3u2u1(_dsrav)
310I_u2u1u3(_dsrl)
311I_u2u1u3(_dsrl32)
312I_u3u2u1(_dsrlv)
313I_u2u1u3(_drotr)
314I_u2u1u3(_drotr32)
315I_u3u1u2(_dsubu)
316I_0(_eret)
317I_u2u1msbdu3(_ext)
318I_u2u1msbu3(_ins)
319I_u1(_j)
320I_u1(_jal)
321I_u2u1(_jalr)
322I_u1(_jr)
323I_u2s3u1(_lb)
324I_u2s3u1(_lbu)
325I_u2s3u1(_ld)
326I_u2s3u1(_lh)
327I_u2s3u1(_lhu)
328I_u2s3u1(_ll)
329I_u2s3u1(_lld)
330I_u1s2(_lui)
331I_u2s3u1(_lw)
332I_u2s3u1(_lwu)
333I_u1u2u3(_mfc0)
334I_u1u2u3(_mfhc0)
335I_u3u1u2(_modu)
336I_u3u1u2(_movn)
337I_u3u1u2(_movz)
338I_u1(_mfhi)
339I_u1(_mflo)
340I_u1u2u3(_mtc0)
341I_u1u2u3(_mthc0)
342I_u1(_mthi)
343I_u1(_mtlo)
344I_u3u1u2(_mul)
345I_u1u2(_multu)
346I_u3u1u2(_mulu)
347I_u3u1u2(_nor)
348I_u3u1u2(_or)
349I_u2u1u3(_ori)
350I_0(_rfe)
351I_u2s3u1(_sb)
352I_u2s3u1(_sc)
353I_u2s3u1(_scd)
354I_u2s3u1(_sd)
355I_u3u1u2(_seleqz)
356I_u3u1u2(_selnez)
357I_u2s3u1(_sh)
358I_u2u1u3(_sll)
359I_u3u2u1(_sllv)
360I_s3s1s2(_slt)
361I_u2u1s3(_slti)
362I_u2u1s3(_sltiu)
363I_u3u1u2(_sltu)
364I_u2u1u3(_sra)
365I_u3u2u1(_srav)
366I_u2u1u3(_srl)
367I_u3u2u1(_srlv)
368I_u2u1u3(_rotr)
369I_u3u1u2(_subu)
370I_u2s3u1(_sw)
371I_u1(_sync)
372I_0(_tlbp)
373I_0(_tlbr)
374I_0(_tlbwi)
375I_0(_tlbwr)
376I_u1(_wait);
377I_u2u1(_wsbh)
378I_u3u1u2(_xor)
379I_u2u1u3(_xori)
380I_u2u1(_yield)
381I_u2u1msbu3(_dins);
382I_u2u1msb32u3(_dinsm);
383I_u2u1msb32msb3(_dinsu);
384I_u1(_syscall);
385I_u1u2s3(_bbit0);
386I_u1u2s3(_bbit1);
387I_u3u1u2(_lwx)
388I_u3u1u2(_ldx)
389I_u1u2(_ldpte)
390I_u2u1u3(_lddir)
391
392#ifdef CONFIG_CPU_CAVIUM_OCTEON
393#include <asm/octeon/octeon.h>
394void uasm_i_pref(u32 **buf, unsigned int a, signed int b,
395 unsigned int c)
396{
397 if (OCTEON_IS_MODEL(OCTEON_CN6XXX) && a <= 24 && a != 5)
398 /*
399 * As per erratum Core-14449, replace prefetches 0-4,
400 * 6-24 with 'pref 28'.
401 */
402 build_insn(buf, insn_pref, c, 28, b);
403 else
404 build_insn(buf, insn_pref, c, a, b);
405}
406UASM_EXPORT_SYMBOL(uasm_i_pref);
407#else
408I_u2s3u1(_pref)
409#endif
410
411/* Handle labels. */
412void uasm_build_label(struct uasm_label **lab, u32 *addr, int lid)
413{
414 (*lab)->addr = addr;
415 (*lab)->lab = lid;
416 (*lab)++;
417}
418UASM_EXPORT_SYMBOL(uasm_build_label);
419
420int uasm_in_compat_space_p(long addr)
421{
422 /* Is this address in 32bit compat space? */
423 return addr == (int)addr;
424}
425UASM_EXPORT_SYMBOL(uasm_in_compat_space_p);
426
427static int uasm_rel_highest(long val)
428{
429#ifdef CONFIG_64BIT
430 return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000;
431#else
432 return 0;
433#endif
434}
435
436static int uasm_rel_higher(long val)
437{
438#ifdef CONFIG_64BIT
439 return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000;
440#else
441 return 0;
442#endif
443}
444
445int uasm_rel_hi(long val)
446{
447 return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000;
448}
449UASM_EXPORT_SYMBOL(uasm_rel_hi);
450
451int uasm_rel_lo(long val)
452{
453 return ((val & 0xffff) ^ 0x8000) - 0x8000;
454}
455UASM_EXPORT_SYMBOL(uasm_rel_lo);
456
457void UASM_i_LA_mostly(u32 **buf, unsigned int rs, long addr)
458{
459 if (!uasm_in_compat_space_p(addr)) {
460 uasm_i_lui(buf, rs, uasm_rel_highest(addr));
461 if (uasm_rel_higher(addr))
462 uasm_i_daddiu(buf, rs, rs, uasm_rel_higher(addr));
463 if (uasm_rel_hi(addr)) {
464 uasm_i_dsll(buf, rs, rs, 16);
465 uasm_i_daddiu(buf, rs, rs,
466 uasm_rel_hi(addr));
467 uasm_i_dsll(buf, rs, rs, 16);
468 } else
469 uasm_i_dsll32(buf, rs, rs, 0);
470 } else
471 uasm_i_lui(buf, rs, uasm_rel_hi(addr));
472}
473UASM_EXPORT_SYMBOL(UASM_i_LA_mostly);
474
475void UASM_i_LA(u32 **buf, unsigned int rs, long addr)
476{
477 UASM_i_LA_mostly(buf, rs, addr);
478 if (uasm_rel_lo(addr)) {
479 if (!uasm_in_compat_space_p(addr))
480 uasm_i_daddiu(buf, rs, rs,
481 uasm_rel_lo(addr));
482 else
483 uasm_i_addiu(buf, rs, rs,
484 uasm_rel_lo(addr));
485 }
486}
487UASM_EXPORT_SYMBOL(UASM_i_LA);
488
489/* Handle relocations. */
490void uasm_r_mips_pc16(struct uasm_reloc **rel, u32 *addr, int lid)
491{
492 (*rel)->addr = addr;
493 (*rel)->type = R_MIPS_PC16;
494 (*rel)->lab = lid;
495 (*rel)++;
496}
497UASM_EXPORT_SYMBOL(uasm_r_mips_pc16);
498
499static inline void __resolve_relocs(struct uasm_reloc *rel,
500 struct uasm_label *lab);
501
502void uasm_resolve_relocs(struct uasm_reloc *rel,
503 struct uasm_label *lab)
504{
505 struct uasm_label *l;
506
507 for (; rel->lab != UASM_LABEL_INVALID; rel++)
508 for (l = lab; l->lab != UASM_LABEL_INVALID; l++)
509 if (rel->lab == l->lab)
510 __resolve_relocs(rel, l);
511}
512UASM_EXPORT_SYMBOL(uasm_resolve_relocs);
513
514void uasm_move_relocs(struct uasm_reloc *rel, u32 *first, u32 *end,
515 long off)
516{
517 for (; rel->lab != UASM_LABEL_INVALID; rel++)
518 if (rel->addr >= first && rel->addr < end)
519 rel->addr += off;
520}
521UASM_EXPORT_SYMBOL(uasm_move_relocs);
522
523void uasm_move_labels(struct uasm_label *lab, u32 *first, u32 *end,
524 long off)
525{
526 for (; lab->lab != UASM_LABEL_INVALID; lab++)
527 if (lab->addr >= first && lab->addr < end)
528 lab->addr += off;
529}
530UASM_EXPORT_SYMBOL(uasm_move_labels);
531
532void uasm_copy_handler(struct uasm_reloc *rel, struct uasm_label *lab,
533 u32 *first, u32 *end, u32 *target)
534{
535 long off = (long)(target - first);
536
537 memcpy(target, first, (end - first) * sizeof(u32));
538
539 uasm_move_relocs(rel, first, end, off);
540 uasm_move_labels(lab, first, end, off);
541}
542UASM_EXPORT_SYMBOL(uasm_copy_handler);
543
544int uasm_insn_has_bdelay(struct uasm_reloc *rel, u32 *addr)
545{
546 for (; rel->lab != UASM_LABEL_INVALID; rel++) {
547 if (rel->addr == addr
548 && (rel->type == R_MIPS_PC16
549 || rel->type == R_MIPS_26))
550 return 1;
551 }
552
553 return 0;
554}
555UASM_EXPORT_SYMBOL(uasm_insn_has_bdelay);
556
557/* Convenience functions for labeled branches. */
558void uasm_il_bltz(u32 **p, struct uasm_reloc **r, unsigned int reg,
559 int lid)
560{
561 uasm_r_mips_pc16(r, *p, lid);
562 uasm_i_bltz(p, reg, 0);
563}
564UASM_EXPORT_SYMBOL(uasm_il_bltz);
565
566void uasm_il_b(u32 **p, struct uasm_reloc **r, int lid)
567{
568 uasm_r_mips_pc16(r, *p, lid);
569 uasm_i_b(p, 0);
570}
571UASM_EXPORT_SYMBOL(uasm_il_b);
572
573void uasm_il_beq(u32 **p, struct uasm_reloc **r, unsigned int r1,
574 unsigned int r2, int lid)
575{
576 uasm_r_mips_pc16(r, *p, lid);
577 uasm_i_beq(p, r1, r2, 0);
578}
579UASM_EXPORT_SYMBOL(uasm_il_beq);
580
581void uasm_il_beqz(u32 **p, struct uasm_reloc **r, unsigned int reg,
582 int lid)
583{
584 uasm_r_mips_pc16(r, *p, lid);
585 uasm_i_beqz(p, reg, 0);
586}
587UASM_EXPORT_SYMBOL(uasm_il_beqz);
588
589void uasm_il_beqzl(u32 **p, struct uasm_reloc **r, unsigned int reg,
590 int lid)
591{
592 uasm_r_mips_pc16(r, *p, lid);
593 uasm_i_beqzl(p, reg, 0);
594}
595UASM_EXPORT_SYMBOL(uasm_il_beqzl);
596
597void uasm_il_bne(u32 **p, struct uasm_reloc **r, unsigned int reg1,
598 unsigned int reg2, int lid)
599{
600 uasm_r_mips_pc16(r, *p, lid);
601 uasm_i_bne(p, reg1, reg2, 0);
602}
603UASM_EXPORT_SYMBOL(uasm_il_bne);
604
605void uasm_il_bnez(u32 **p, struct uasm_reloc **r, unsigned int reg,
606 int lid)
607{
608 uasm_r_mips_pc16(r, *p, lid);
609 uasm_i_bnez(p, reg, 0);
610}
611UASM_EXPORT_SYMBOL(uasm_il_bnez);
612
613void uasm_il_bgezl(u32 **p, struct uasm_reloc **r, unsigned int reg,
614 int lid)
615{
616 uasm_r_mips_pc16(r, *p, lid);
617 uasm_i_bgezl(p, reg, 0);
618}
619UASM_EXPORT_SYMBOL(uasm_il_bgezl);
620
621void uasm_il_bgez(u32 **p, struct uasm_reloc **r, unsigned int reg,
622 int lid)
623{
624 uasm_r_mips_pc16(r, *p, lid);
625 uasm_i_bgez(p, reg, 0);
626}
627UASM_EXPORT_SYMBOL(uasm_il_bgez);
628
629void uasm_il_bbit0(u32 **p, struct uasm_reloc **r, unsigned int reg,
630 unsigned int bit, int lid)
631{
632 uasm_r_mips_pc16(r, *p, lid);
633 uasm_i_bbit0(p, reg, bit, 0);
634}
635UASM_EXPORT_SYMBOL(uasm_il_bbit0);
636
637void uasm_il_bbit1(u32 **p, struct uasm_reloc **r, unsigned int reg,
638 unsigned int bit, int lid)
639{
640 uasm_r_mips_pc16(r, *p, lid);
641 uasm_i_bbit1(p, reg, bit, 0);
642}
643UASM_EXPORT_SYMBOL(uasm_il_bbit1);
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-04-16 09:27:28 +0800