aboutsummaryrefslogtreecommitdiffstats
path: root/arch/mips/lib
diff options
context:
space:
mode:
authorWe-unite <3205135446@qq.com>2025-03-08 22:04:20 +0800
committerWe-unite <3205135446@qq.com>2025-03-08 22:04:20 +0800
commita07bb8fd1299070229f0e8f3dcb57ffd5ef9870a (patch)
tree84f21bd0bf7071bc5fc7dd989e77d7ceb5476682 /arch/mips/lib
downloadohosKernel-a07bb8fd1299070229f0e8f3dcb57ffd5ef9870a.tar.gz
ohosKernel-a07bb8fd1299070229f0e8f3dcb57ffd5ef9870a.zip
Initial commit: OpenHarmony-v4.0-ReleaseOpenHarmony-v4.0-Release
Diffstat (limited to 'arch/mips/lib')
-rw-r--r--arch/mips/lib/Makefile19
-rw-r--r--arch/mips/lib/bitops.c148
-rw-r--r--arch/mips/lib/bswapdi.c17
-rw-r--r--arch/mips/lib/bswapsi.c13
-rw-r--r--arch/mips/lib/csum_partial.S754
-rw-r--r--arch/mips/lib/delay.c69
-rw-r--r--arch/mips/lib/dump_tlb.c199
-rw-r--r--arch/mips/lib/iomap-pci.c46
-rw-r--r--arch/mips/lib/iomap_copy.c29
-rw-r--r--arch/mips/lib/libgcc.h43
-rw-r--r--arch/mips/lib/memcpy.S709
-rw-r--r--arch/mips/lib/memset.S328
-rw-r--r--arch/mips/lib/mips-atomic.c113
-rw-r--r--arch/mips/lib/multi3.c54
-rw-r--r--arch/mips/lib/r3k_dump_tlb.c75
-rw-r--r--arch/mips/lib/strncpy_user.S85
-rw-r--r--arch/mips/lib/strnlen_user.S84
-rw-r--r--arch/mips/lib/uncached.c82
18 files changed, 2867 insertions, 0 deletions
diff --git a/arch/mips/lib/Makefile b/arch/mips/lib/Makefile
new file mode 100644
index 000000000..479f50559
--- /dev/null
+++ b/arch/mips/lib/Makefile
@@ -0,0 +1,19 @@
1# SPDX-License-Identifier: GPL-2.0
2#
3# Makefile for MIPS-specific library files..
4#
5
6lib-y += bitops.o csum_partial.o delay.o memcpy.o memset.o \
7 mips-atomic.o strncpy_user.o \
8 strnlen_user.o uncached.o
9
10obj-y += iomap_copy.o
11obj-$(CONFIG_PCI) += iomap-pci.o
12lib-$(CONFIG_GENERIC_CSUM) := $(filter-out csum_partial.o, $(lib-y))
13
14obj-$(CONFIG_CPU_GENERIC_DUMP_TLB) += dump_tlb.o
15obj-$(CONFIG_CPU_R3000) += r3k_dump_tlb.o
16obj-$(CONFIG_CPU_TX39XX) += r3k_dump_tlb.o
17
18# libgcc-style stuff needed in the kernel
19obj-y += bswapsi.o bswapdi.o multi3.o
diff --git a/arch/mips/lib/bitops.c b/arch/mips/lib/bitops.c
new file mode 100644
index 000000000..116d0bd8b
--- /dev/null
+++ b/arch/mips/lib/bitops.c
@@ -0,0 +1,148 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 1994-1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (c) 1999, 2000 Silicon Graphics, Inc.
8 */
9#include <linux/bitops.h>
10#include <linux/bits.h>
11#include <linux/irqflags.h>
12#include <linux/export.h>
13
14
15/**
16 * __mips_set_bit - Atomically set a bit in memory. This is called by
17 * set_bit() if it cannot find a faster solution.
18 * @nr: the bit to set
19 * @addr: the address to start counting from
20 */
21void __mips_set_bit(unsigned long nr, volatile unsigned long *addr)
22{
23 volatile unsigned long *a = &addr[BIT_WORD(nr)];
24 unsigned int bit = nr % BITS_PER_LONG;
25 unsigned long mask;
26 unsigned long flags;
27
28 mask = 1UL << bit;
29 raw_local_irq_save(flags);
30 *a |= mask;
31 raw_local_irq_restore(flags);
32}
33EXPORT_SYMBOL(__mips_set_bit);
34
35
36/**
37 * __mips_clear_bit - Clears a bit in memory. This is called by clear_bit() if
38 * it cannot find a faster solution.
39 * @nr: Bit to clear
40 * @addr: Address to start counting from
41 */
42void __mips_clear_bit(unsigned long nr, volatile unsigned long *addr)
43{
44 volatile unsigned long *a = &addr[BIT_WORD(nr)];
45 unsigned int bit = nr % BITS_PER_LONG;
46 unsigned long mask;
47 unsigned long flags;
48
49 mask = 1UL << bit;
50 raw_local_irq_save(flags);
51 *a &= ~mask;
52 raw_local_irq_restore(flags);
53}
54EXPORT_SYMBOL(__mips_clear_bit);
55
56
57/**
58 * __mips_change_bit - Toggle a bit in memory. This is called by change_bit()
59 * if it cannot find a faster solution.
60 * @nr: Bit to change
61 * @addr: Address to start counting from
62 */
63void __mips_change_bit(unsigned long nr, volatile unsigned long *addr)
64{
65 volatile unsigned long *a = &addr[BIT_WORD(nr)];
66 unsigned int bit = nr % BITS_PER_LONG;
67 unsigned long mask;
68 unsigned long flags;
69
70 mask = 1UL << bit;
71 raw_local_irq_save(flags);
72 *a ^= mask;
73 raw_local_irq_restore(flags);
74}
75EXPORT_SYMBOL(__mips_change_bit);
76
77
78/**
79 * __mips_test_and_set_bit_lock - Set a bit and return its old value. This is
80 * called by test_and_set_bit_lock() if it cannot find a faster solution.
81 * @nr: Bit to set
82 * @addr: Address to count from
83 */
84int __mips_test_and_set_bit_lock(unsigned long nr,
85 volatile unsigned long *addr)
86{
87 volatile unsigned long *a = &addr[BIT_WORD(nr)];
88 unsigned int bit = nr % BITS_PER_LONG;
89 unsigned long mask;
90 unsigned long flags;
91 int res;
92
93 mask = 1UL << bit;
94 raw_local_irq_save(flags);
95 res = (mask & *a) != 0;
96 *a |= mask;
97 raw_local_irq_restore(flags);
98 return res;
99}
100EXPORT_SYMBOL(__mips_test_and_set_bit_lock);
101
102
103/**
104 * __mips_test_and_clear_bit - Clear a bit and return its old value. This is
105 * called by test_and_clear_bit() if it cannot find a faster solution.
106 * @nr: Bit to clear
107 * @addr: Address to count from
108 */
109int __mips_test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
110{
111 volatile unsigned long *a = &addr[BIT_WORD(nr)];
112 unsigned int bit = nr % BITS_PER_LONG;
113 unsigned long mask;
114 unsigned long flags;
115 int res;
116
117 mask = 1UL << bit;
118 raw_local_irq_save(flags);
119 res = (mask & *a) != 0;
120 *a &= ~mask;
121 raw_local_irq_restore(flags);
122 return res;
123}
124EXPORT_SYMBOL(__mips_test_and_clear_bit);
125
126
127/**
128 * __mips_test_and_change_bit - Change a bit and return its old value. This is
129 * called by test_and_change_bit() if it cannot find a faster solution.
130 * @nr: Bit to change
131 * @addr: Address to count from
132 */
133int __mips_test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
134{
135 volatile unsigned long *a = &addr[BIT_WORD(nr)];
136 unsigned int bit = nr % BITS_PER_LONG;
137 unsigned long mask;
138 unsigned long flags;
139 int res;
140
141 mask = 1UL << bit;
142 raw_local_irq_save(flags);
143 res = (mask & *a) != 0;
144 *a ^= mask;
145 raw_local_irq_restore(flags);
146 return res;
147}
148EXPORT_SYMBOL(__mips_test_and_change_bit);
diff --git a/arch/mips/lib/bswapdi.c b/arch/mips/lib/bswapdi.c
new file mode 100644
index 000000000..fcef74084
--- /dev/null
+++ b/arch/mips/lib/bswapdi.c
@@ -0,0 +1,17 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/export.h>
3#include <linux/compiler.h>
4
5unsigned long long notrace __bswapdi2(unsigned long long u)
6{
7 return (((u) & 0xff00000000000000ull) >> 56) |
8 (((u) & 0x00ff000000000000ull) >> 40) |
9 (((u) & 0x0000ff0000000000ull) >> 24) |
10 (((u) & 0x000000ff00000000ull) >> 8) |
11 (((u) & 0x00000000ff000000ull) << 8) |
12 (((u) & 0x0000000000ff0000ull) << 24) |
13 (((u) & 0x000000000000ff00ull) << 40) |
14 (((u) & 0x00000000000000ffull) << 56);
15}
16
17EXPORT_SYMBOL(__bswapdi2);
diff --git a/arch/mips/lib/bswapsi.c b/arch/mips/lib/bswapsi.c
new file mode 100644
index 000000000..22d8e4f6d
--- /dev/null
+++ b/arch/mips/lib/bswapsi.c
@@ -0,0 +1,13 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/export.h>
3#include <linux/compiler.h>
4
5unsigned int notrace __bswapsi2(unsigned int u)
6{
7 return (((u) & 0xff000000) >> 24) |
8 (((u) & 0x00ff0000) >> 8) |
9 (((u) & 0x0000ff00) << 8) |
10 (((u) & 0x000000ff) << 24);
11}
12
13EXPORT_SYMBOL(__bswapsi2);
diff --git a/arch/mips/lib/csum_partial.S b/arch/mips/lib/csum_partial.S
new file mode 100644
index 000000000..a46db0807
--- /dev/null
+++ b/arch/mips/lib/csum_partial.S
@@ -0,0 +1,754 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Quick'n'dirty IP checksum ...
7 *
8 * Copyright (C) 1998, 1999 Ralf Baechle
9 * Copyright (C) 1999 Silicon Graphics, Inc.
10 * Copyright (C) 2007 Maciej W. Rozycki
11 * Copyright (C) 2014 Imagination Technologies Ltd.
12 */
13#include <linux/errno.h>
14#include <asm/asm.h>
15#include <asm/asm-offsets.h>
16#include <asm/export.h>
17#include <asm/regdef.h>
18
19#ifdef CONFIG_64BIT
20/*
21 * As we are sharing code base with the mips32 tree (which use the o32 ABI
22 * register definitions). We need to redefine the register definitions from
23 * the n64 ABI register naming to the o32 ABI register naming.
24 */
25#undef t0
26#undef t1
27#undef t2
28#undef t3
29#define t0 $8
30#define t1 $9
31#define t2 $10
32#define t3 $11
33#define t4 $12
34#define t5 $13
35#define t6 $14
36#define t7 $15
37
38#define USE_DOUBLE
39#endif
40
41#ifdef USE_DOUBLE
42
43#define LOAD ld
44#define LOAD32 lwu
45#define ADD daddu
46#define NBYTES 8
47
48#else
49
50#define LOAD lw
51#define LOAD32 lw
52#define ADD addu
53#define NBYTES 4
54
55#endif /* USE_DOUBLE */
56
57#define UNIT(unit) ((unit)*NBYTES)
58
59#define ADDC(sum,reg) \
60 .set push; \
61 .set noat; \
62 ADD sum, reg; \
63 sltu v1, sum, reg; \
64 ADD sum, v1; \
65 .set pop
66
67#define ADDC32(sum,reg) \
68 .set push; \
69 .set noat; \
70 addu sum, reg; \
71 sltu v1, sum, reg; \
72 addu sum, v1; \
73 .set pop
74
75#define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) \
76 LOAD _t0, (offset + UNIT(0))(src); \
77 LOAD _t1, (offset + UNIT(1))(src); \
78 LOAD _t2, (offset + UNIT(2))(src); \
79 LOAD _t3, (offset + UNIT(3))(src); \
80 ADDC(_t0, _t1); \
81 ADDC(_t2, _t3); \
82 ADDC(sum, _t0); \
83 ADDC(sum, _t2)
84
85#ifdef USE_DOUBLE
86#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
87 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3)
88#else
89#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
90 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3); \
91 CSUM_BIGCHUNK1(src, offset + 0x10, sum, _t0, _t1, _t2, _t3)
92#endif
93
94/*
95 * a0: source address
96 * a1: length of the area to checksum
97 * a2: partial checksum
98 */
99
100#define src a0
101#define sum v0
102
103 .text
104 .set noreorder
105 .align 5
106LEAF(csum_partial)
107EXPORT_SYMBOL(csum_partial)
108 move sum, zero
109 move t7, zero
110
111 sltiu t8, a1, 0x8
112 bnez t8, .Lsmall_csumcpy /* < 8 bytes to copy */
113 move t2, a1
114
115 andi t7, src, 0x1 /* odd buffer? */
116
117.Lhword_align:
118 beqz t7, .Lword_align
119 andi t8, src, 0x2
120
121 lbu t0, (src)
122 LONG_SUBU a1, a1, 0x1
123#ifdef __MIPSEL__
124 sll t0, t0, 8
125#endif
126 ADDC(sum, t0)
127 PTR_ADDU src, src, 0x1
128 andi t8, src, 0x2
129
130.Lword_align:
131 beqz t8, .Ldword_align
132 sltiu t8, a1, 56
133
134 lhu t0, (src)
135 LONG_SUBU a1, a1, 0x2
136 ADDC(sum, t0)
137 sltiu t8, a1, 56
138 PTR_ADDU src, src, 0x2
139
140.Ldword_align:
141 bnez t8, .Ldo_end_words
142 move t8, a1
143
144 andi t8, src, 0x4
145 beqz t8, .Lqword_align
146 andi t8, src, 0x8
147
148 LOAD32 t0, 0x00(src)
149 LONG_SUBU a1, a1, 0x4
150 ADDC(sum, t0)
151 PTR_ADDU src, src, 0x4
152 andi t8, src, 0x8
153
154.Lqword_align:
155 beqz t8, .Loword_align
156 andi t8, src, 0x10
157
158#ifdef USE_DOUBLE
159 ld t0, 0x00(src)
160 LONG_SUBU a1, a1, 0x8
161 ADDC(sum, t0)
162#else
163 lw t0, 0x00(src)
164 lw t1, 0x04(src)
165 LONG_SUBU a1, a1, 0x8
166 ADDC(sum, t0)
167 ADDC(sum, t1)
168#endif
169 PTR_ADDU src, src, 0x8
170 andi t8, src, 0x10
171
172.Loword_align:
173 beqz t8, .Lbegin_movement
174 LONG_SRL t8, a1, 0x7
175
176#ifdef USE_DOUBLE
177 ld t0, 0x00(src)
178 ld t1, 0x08(src)
179 ADDC(sum, t0)
180 ADDC(sum, t1)
181#else
182 CSUM_BIGCHUNK1(src, 0x00, sum, t0, t1, t3, t4)
183#endif
184 LONG_SUBU a1, a1, 0x10
185 PTR_ADDU src, src, 0x10
186 LONG_SRL t8, a1, 0x7
187
188.Lbegin_movement:
189 beqz t8, 1f
190 andi t2, a1, 0x40
191
192.Lmove_128bytes:
193 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
194 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
195 CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4)
196 CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4)
197 LONG_SUBU t8, t8, 0x01
198 .set reorder /* DADDI_WAR */
199 PTR_ADDU src, src, 0x80
200 bnez t8, .Lmove_128bytes
201 .set noreorder
202
2031:
204 beqz t2, 1f
205 andi t2, a1, 0x20
206
207.Lmove_64bytes:
208 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
209 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
210 PTR_ADDU src, src, 0x40
211
2121:
213 beqz t2, .Ldo_end_words
214 andi t8, a1, 0x1c
215
216.Lmove_32bytes:
217 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
218 andi t8, a1, 0x1c
219 PTR_ADDU src, src, 0x20
220
221.Ldo_end_words:
222 beqz t8, .Lsmall_csumcpy
223 andi t2, a1, 0x3
224 LONG_SRL t8, t8, 0x2
225
226.Lend_words:
227 LOAD32 t0, (src)
228 LONG_SUBU t8, t8, 0x1
229 ADDC(sum, t0)
230 .set reorder /* DADDI_WAR */
231 PTR_ADDU src, src, 0x4
232 bnez t8, .Lend_words
233 .set noreorder
234
235/* unknown src alignment and < 8 bytes to go */
236.Lsmall_csumcpy:
237 move a1, t2
238
239 andi t0, a1, 4
240 beqz t0, 1f
241 andi t0, a1, 2
242
243 /* Still a full word to go */
244 ulw t1, (src)
245 PTR_ADDIU src, 4
246#ifdef USE_DOUBLE
247 dsll t1, t1, 32 /* clear lower 32bit */
248#endif
249 ADDC(sum, t1)
250
2511: move t1, zero
252 beqz t0, 1f
253 andi t0, a1, 1
254
255 /* Still a halfword to go */
256 ulhu t1, (src)
257 PTR_ADDIU src, 2
258
2591: beqz t0, 1f
260 sll t1, t1, 16
261
262 lbu t2, (src)
263 nop
264
265#ifdef __MIPSEB__
266 sll t2, t2, 8
267#endif
268 or t1, t2
269
2701: ADDC(sum, t1)
271
272 /* fold checksum */
273#ifdef USE_DOUBLE
274 dsll32 v1, sum, 0
275 daddu sum, v1
276 sltu v1, sum, v1
277 dsra32 sum, sum, 0
278 addu sum, v1
279#endif
280
281 /* odd buffer alignment? */
282#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \
283 defined(CONFIG_CPU_LOONGSON64)
284 .set push
285 .set arch=mips32r2
286 wsbh v1, sum
287 movn sum, v1, t7
288 .set pop
289#else
290 beqz t7, 1f /* odd buffer alignment? */
291 lui v1, 0x00ff
292 addu v1, 0x00ff
293 and t0, sum, v1
294 sll t0, t0, 8
295 srl sum, sum, 8
296 and sum, sum, v1
297 or sum, sum, t0
2981:
299#endif
300 .set reorder
301 /* Add the passed partial csum. */
302 ADDC32(sum, a2)
303 jr ra
304 .set noreorder
305 END(csum_partial)
306
307
308/*
309 * checksum and copy routines based on memcpy.S
310 *
311 * csum_partial_copy_nocheck(src, dst, len)
312 * __csum_partial_copy_kernel(src, dst, len)
313 *
314 * See "Spec" in memcpy.S for details. Unlike __copy_user, all
315 * function in this file use the standard calling convention.
316 */
317
318#define src a0
319#define dst a1
320#define len a2
321#define sum v0
322#define odd t8
323
324/*
325 * All exception handlers simply return 0.
326 */
327
328/* Instruction type */
329#define LD_INSN 1
330#define ST_INSN 2
331#define LEGACY_MODE 1
332#define EVA_MODE 2
333#define USEROP 1
334#define KERNELOP 2
335
336/*
337 * Wrapper to add an entry in the exception table
338 * in case the insn causes a memory exception.
339 * Arguments:
340 * insn : Load/store instruction
341 * type : Instruction type
342 * reg : Register
343 * addr : Address
344 * handler : Exception handler
345 */
346#define EXC(insn, type, reg, addr) \
347 .if \mode == LEGACY_MODE; \
3489: insn reg, addr; \
349 .section __ex_table,"a"; \
350 PTR 9b, .L_exc; \
351 .previous; \
352 /* This is enabled in EVA mode */ \
353 .else; \
354 /* If loading from user or storing to user */ \
355 .if ((\from == USEROP) && (type == LD_INSN)) || \
356 ((\to == USEROP) && (type == ST_INSN)); \
3579: __BUILD_EVA_INSN(insn##e, reg, addr); \
358 .section __ex_table,"a"; \
359 PTR 9b, .L_exc; \
360 .previous; \
361 .else; \
362 /* EVA without exception */ \
363 insn reg, addr; \
364 .endif; \
365 .endif
366
367#undef LOAD
368
369#ifdef USE_DOUBLE
370
371#define LOADK ld /* No exception */
372#define LOAD(reg, addr) EXC(ld, LD_INSN, reg, addr)
373#define LOADBU(reg, addr) EXC(lbu, LD_INSN, reg, addr)
374#define LOADL(reg, addr) EXC(ldl, LD_INSN, reg, addr)
375#define LOADR(reg, addr) EXC(ldr, LD_INSN, reg, addr)
376#define STOREB(reg, addr) EXC(sb, ST_INSN, reg, addr)
377#define STOREL(reg, addr) EXC(sdl, ST_INSN, reg, addr)
378#define STORER(reg, addr) EXC(sdr, ST_INSN, reg, addr)
379#define STORE(reg, addr) EXC(sd, ST_INSN, reg, addr)
380#define ADD daddu
381#define SUB dsubu
382#define SRL dsrl
383#define SLL dsll
384#define SLLV dsllv
385#define SRLV dsrlv
386#define NBYTES 8
387#define LOG_NBYTES 3
388
389#else
390
391#define LOADK lw /* No exception */
392#define LOAD(reg, addr) EXC(lw, LD_INSN, reg, addr)
393#define LOADBU(reg, addr) EXC(lbu, LD_INSN, reg, addr)
394#define LOADL(reg, addr) EXC(lwl, LD_INSN, reg, addr)
395#define LOADR(reg, addr) EXC(lwr, LD_INSN, reg, addr)
396#define STOREB(reg, addr) EXC(sb, ST_INSN, reg, addr)
397#define STOREL(reg, addr) EXC(swl, ST_INSN, reg, addr)
398#define STORER(reg, addr) EXC(swr, ST_INSN, reg, addr)
399#define STORE(reg, addr) EXC(sw, ST_INSN, reg, addr)
400#define ADD addu
401#define SUB subu
402#define SRL srl
403#define SLL sll
404#define SLLV sllv
405#define SRLV srlv
406#define NBYTES 4
407#define LOG_NBYTES 2
408
409#endif /* USE_DOUBLE */
410
411#ifdef CONFIG_CPU_LITTLE_ENDIAN
412#define LDFIRST LOADR
413#define LDREST LOADL
414#define STFIRST STORER
415#define STREST STOREL
416#define SHIFT_DISCARD SLLV
417#define SHIFT_DISCARD_REVERT SRLV
418#else
419#define LDFIRST LOADL
420#define LDREST LOADR
421#define STFIRST STOREL
422#define STREST STORER
423#define SHIFT_DISCARD SRLV
424#define SHIFT_DISCARD_REVERT SLLV
425#endif
426
427#define FIRST(unit) ((unit)*NBYTES)
428#define REST(unit) (FIRST(unit)+NBYTES-1)
429
430#define ADDRMASK (NBYTES-1)
431
432#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
433 .set noat
434#else
435 .set at=v1
436#endif
437
438 .macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to
439
440 li sum, -1
441 move odd, zero
442 /*
443 * Note: dst & src may be unaligned, len may be 0
444 * Temps
445 */
446 /*
447 * The "issue break"s below are very approximate.
448 * Issue delays for dcache fills will perturb the schedule, as will
449 * load queue full replay traps, etc.
450 *
451 * If len < NBYTES use byte operations.
452 */
453 sltu t2, len, NBYTES
454 and t1, dst, ADDRMASK
455 bnez t2, .Lcopy_bytes_checklen\@
456 and t0, src, ADDRMASK
457 andi odd, dst, 0x1 /* odd buffer? */
458 bnez t1, .Ldst_unaligned\@
459 nop
460 bnez t0, .Lsrc_unaligned_dst_aligned\@
461 /*
462 * use delay slot for fall-through
463 * src and dst are aligned; need to compute rem
464 */
465.Lboth_aligned\@:
466 SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter
467 beqz t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES
468 nop
469 SUB len, 8*NBYTES # subtract here for bgez loop
470 .align 4
4711:
472 LOAD(t0, UNIT(0)(src))
473 LOAD(t1, UNIT(1)(src))
474 LOAD(t2, UNIT(2)(src))
475 LOAD(t3, UNIT(3)(src))
476 LOAD(t4, UNIT(4)(src))
477 LOAD(t5, UNIT(5)(src))
478 LOAD(t6, UNIT(6)(src))
479 LOAD(t7, UNIT(7)(src))
480 SUB len, len, 8*NBYTES
481 ADD src, src, 8*NBYTES
482 STORE(t0, UNIT(0)(dst))
483 ADDC(t0, t1)
484 STORE(t1, UNIT(1)(dst))
485 ADDC(sum, t0)
486 STORE(t2, UNIT(2)(dst))
487 ADDC(t2, t3)
488 STORE(t3, UNIT(3)(dst))
489 ADDC(sum, t2)
490 STORE(t4, UNIT(4)(dst))
491 ADDC(t4, t5)
492 STORE(t5, UNIT(5)(dst))
493 ADDC(sum, t4)
494 STORE(t6, UNIT(6)(dst))
495 ADDC(t6, t7)
496 STORE(t7, UNIT(7)(dst))
497 ADDC(sum, t6)
498 .set reorder /* DADDI_WAR */
499 ADD dst, dst, 8*NBYTES
500 bgez len, 1b
501 .set noreorder
502 ADD len, 8*NBYTES # revert len (see above)
503
504 /*
505 * len == the number of bytes left to copy < 8*NBYTES
506 */
507.Lcleanup_both_aligned\@:
508#define rem t7
509 beqz len, .Ldone\@
510 sltu t0, len, 4*NBYTES
511 bnez t0, .Lless_than_4units\@
512 and rem, len, (NBYTES-1) # rem = len % NBYTES
513 /*
514 * len >= 4*NBYTES
515 */
516 LOAD(t0, UNIT(0)(src))
517 LOAD(t1, UNIT(1)(src))
518 LOAD(t2, UNIT(2)(src))
519 LOAD(t3, UNIT(3)(src))
520 SUB len, len, 4*NBYTES
521 ADD src, src, 4*NBYTES
522 STORE(t0, UNIT(0)(dst))
523 ADDC(t0, t1)
524 STORE(t1, UNIT(1)(dst))
525 ADDC(sum, t0)
526 STORE(t2, UNIT(2)(dst))
527 ADDC(t2, t3)
528 STORE(t3, UNIT(3)(dst))
529 ADDC(sum, t2)
530 .set reorder /* DADDI_WAR */
531 ADD dst, dst, 4*NBYTES
532 beqz len, .Ldone\@
533 .set noreorder
534.Lless_than_4units\@:
535 /*
536 * rem = len % NBYTES
537 */
538 beq rem, len, .Lcopy_bytes\@
539 nop
5401:
541 LOAD(t0, 0(src))
542 ADD src, src, NBYTES
543 SUB len, len, NBYTES
544 STORE(t0, 0(dst))
545 ADDC(sum, t0)
546 .set reorder /* DADDI_WAR */
547 ADD dst, dst, NBYTES
548 bne rem, len, 1b
549 .set noreorder
550
551 /*
552 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
553 * A loop would do only a byte at a time with possible branch
554 * mispredicts. Can't do an explicit LOAD dst,mask,or,STORE
555 * because can't assume read-access to dst. Instead, use
556 * STREST dst, which doesn't require read access to dst.
557 *
558 * This code should perform better than a simple loop on modern,
559 * wide-issue mips processors because the code has fewer branches and
560 * more instruction-level parallelism.
561 */
562#define bits t2
563 beqz len, .Ldone\@
564 ADD t1, dst, len # t1 is just past last byte of dst
565 li bits, 8*NBYTES
566 SLL rem, len, 3 # rem = number of bits to keep
567 LOAD(t0, 0(src))
568 SUB bits, bits, rem # bits = number of bits to discard
569 SHIFT_DISCARD t0, t0, bits
570 STREST(t0, -1(t1))
571 SHIFT_DISCARD_REVERT t0, t0, bits
572 .set reorder
573 ADDC(sum, t0)
574 b .Ldone\@
575 .set noreorder
576.Ldst_unaligned\@:
577 /*
578 * dst is unaligned
579 * t0 = src & ADDRMASK
580 * t1 = dst & ADDRMASK; T1 > 0
581 * len >= NBYTES
582 *
583 * Copy enough bytes to align dst
584 * Set match = (src and dst have same alignment)
585 */
586#define match rem
587 LDFIRST(t3, FIRST(0)(src))
588 ADD t2, zero, NBYTES
589 LDREST(t3, REST(0)(src))
590 SUB t2, t2, t1 # t2 = number of bytes copied
591 xor match, t0, t1
592 STFIRST(t3, FIRST(0)(dst))
593 SLL t4, t1, 3 # t4 = number of bits to discard
594 SHIFT_DISCARD t3, t3, t4
595 /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
596 ADDC(sum, t3)
597 beq len, t2, .Ldone\@
598 SUB len, len, t2
599 ADD dst, dst, t2
600 beqz match, .Lboth_aligned\@
601 ADD src, src, t2
602
603.Lsrc_unaligned_dst_aligned\@:
604 SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter
605 beqz t0, .Lcleanup_src_unaligned\@
606 and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES
6071:
608/*
609 * Avoid consecutive LD*'s to the same register since some mips
610 * implementations can't issue them in the same cycle.
611 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
612 * are to the same unit (unless src is aligned, but it's not).
613 */
614 LDFIRST(t0, FIRST(0)(src))
615 LDFIRST(t1, FIRST(1)(src))
616 SUB len, len, 4*NBYTES
617 LDREST(t0, REST(0)(src))
618 LDREST(t1, REST(1)(src))
619 LDFIRST(t2, FIRST(2)(src))
620 LDFIRST(t3, FIRST(3)(src))
621 LDREST(t2, REST(2)(src))
622 LDREST(t3, REST(3)(src))
623 ADD src, src, 4*NBYTES
624#ifdef CONFIG_CPU_SB1
625 nop # improves slotting
626#endif
627 STORE(t0, UNIT(0)(dst))
628 ADDC(t0, t1)
629 STORE(t1, UNIT(1)(dst))
630 ADDC(sum, t0)
631 STORE(t2, UNIT(2)(dst))
632 ADDC(t2, t3)
633 STORE(t3, UNIT(3)(dst))
634 ADDC(sum, t2)
635 .set reorder /* DADDI_WAR */
636 ADD dst, dst, 4*NBYTES
637 bne len, rem, 1b
638 .set noreorder
639
640.Lcleanup_src_unaligned\@:
641 beqz len, .Ldone\@
642 and rem, len, NBYTES-1 # rem = len % NBYTES
643 beq rem, len, .Lcopy_bytes\@
644 nop
6451:
646 LDFIRST(t0, FIRST(0)(src))
647 LDREST(t0, REST(0)(src))
648 ADD src, src, NBYTES
649 SUB len, len, NBYTES
650 STORE(t0, 0(dst))
651 ADDC(sum, t0)
652 .set reorder /* DADDI_WAR */
653 ADD dst, dst, NBYTES
654 bne len, rem, 1b
655 .set noreorder
656
657.Lcopy_bytes_checklen\@:
658 beqz len, .Ldone\@
659 nop
660.Lcopy_bytes\@:
661 /* 0 < len < NBYTES */
662#ifdef CONFIG_CPU_LITTLE_ENDIAN
663#define SHIFT_START 0
664#define SHIFT_INC 8
665#else
666#define SHIFT_START 8*(NBYTES-1)
667#define SHIFT_INC -8
668#endif
669 move t2, zero # partial word
670 li t3, SHIFT_START # shift
671#define COPY_BYTE(N) \
672 LOADBU(t0, N(src)); \
673 SUB len, len, 1; \
674 STOREB(t0, N(dst)); \
675 SLLV t0, t0, t3; \
676 addu t3, SHIFT_INC; \
677 beqz len, .Lcopy_bytes_done\@; \
678 or t2, t0
679
680 COPY_BYTE(0)
681 COPY_BYTE(1)
682#ifdef USE_DOUBLE
683 COPY_BYTE(2)
684 COPY_BYTE(3)
685 COPY_BYTE(4)
686 COPY_BYTE(5)
687#endif
688 LOADBU(t0, NBYTES-2(src))
689 SUB len, len, 1
690 STOREB(t0, NBYTES-2(dst))
691 SLLV t0, t0, t3
692 or t2, t0
693.Lcopy_bytes_done\@:
694 ADDC(sum, t2)
695.Ldone\@:
696 /* fold checksum */
697 .set push
698 .set noat
699#ifdef USE_DOUBLE
700 dsll32 v1, sum, 0
701 daddu sum, v1
702 sltu v1, sum, v1
703 dsra32 sum, sum, 0
704 addu sum, v1
705#endif
706
707#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \
708 defined(CONFIG_CPU_LOONGSON64)
709 .set push
710 .set arch=mips32r2
711 wsbh v1, sum
712 movn sum, v1, odd
713 .set pop
714#else
715 beqz odd, 1f /* odd buffer alignment? */
716 lui v1, 0x00ff
717 addu v1, 0x00ff
718 and t0, sum, v1
719 sll t0, t0, 8
720 srl sum, sum, 8
721 and sum, sum, v1
722 or sum, sum, t0
7231:
724#endif
725 .set pop
726 .set reorder
727 jr ra
728 .set noreorder
729 .endm
730
731 .set noreorder
732.L_exc:
733 jr ra
734 li v0, 0
735
736FEXPORT(__csum_partial_copy_nocheck)
737EXPORT_SYMBOL(__csum_partial_copy_nocheck)
738#ifndef CONFIG_EVA
739FEXPORT(__csum_partial_copy_to_user)
740EXPORT_SYMBOL(__csum_partial_copy_to_user)
741FEXPORT(__csum_partial_copy_from_user)
742EXPORT_SYMBOL(__csum_partial_copy_from_user)
743#endif
744__BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP
745
746#ifdef CONFIG_EVA
747LEAF(__csum_partial_copy_to_user)
748__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE KERNELOP USEROP
749END(__csum_partial_copy_to_user)
750
751LEAF(__csum_partial_copy_from_user)
752__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE USEROP KERNELOP
753END(__csum_partial_copy_from_user)
754#endif
diff --git a/arch/mips/lib/delay.c b/arch/mips/lib/delay.c
new file mode 100644
index 000000000..2e8dfc1d5
--- /dev/null
+++ b/arch/mips/lib/delay.c
@@ -0,0 +1,69 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1994 by Waldorf Electronics
7 * Copyright (C) 1995 - 2000, 01, 03 by Ralf Baechle
8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
9 * Copyright (C) 2007, 2014 Maciej W. Rozycki
10 */
11#include <linux/delay.h>
12#include <linux/export.h>
13#include <linux/param.h>
14#include <linux/smp.h>
15#include <linux/stringify.h>
16
17#include <asm/asm.h>
18#include <asm/compiler.h>
19#include <asm/war.h>
20
21#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
22#define GCC_DADDI_IMM_ASM() "I"
23#else
24#define GCC_DADDI_IMM_ASM() "r"
25#endif
26
27#ifndef CONFIG_HAVE_PLAT_DELAY
28
29void __delay(unsigned long loops)
30{
31 __asm__ __volatile__ (
32 " .set noreorder \n"
33 " .align 3 \n"
34 "1: bnez %0, 1b \n"
35 " " __stringify(LONG_SUBU) " %0, %1 \n"
36 " .set reorder \n"
37 : "=r" (loops)
38 : GCC_DADDI_IMM_ASM() (1), "0" (loops));
39}
40EXPORT_SYMBOL(__delay);
41
42/*
43 * Division by multiplication: you don't have to worry about
44 * loss of precision.
45 *
46 * Use only for very small delays ( < 1 msec). Should probably use a
47 * lookup table, really, as the multiplications take much too long with
48 * short delays. This is a "reasonable" implementation, though (and the
49 * first constant multiplications gets optimized away if the delay is
50 * a constant)
51 */
52
53void __udelay(unsigned long us)
54{
55 unsigned int lpj = raw_current_cpu_data.udelay_val;
56
57 __delay((us * 0x000010c7ull * HZ * lpj) >> 32);
58}
59EXPORT_SYMBOL(__udelay);
60
61void __ndelay(unsigned long ns)
62{
63 unsigned int lpj = raw_current_cpu_data.udelay_val;
64
65 __delay((ns * 0x00000005ull * HZ * lpj) >> 32);
66}
67EXPORT_SYMBOL(__ndelay);
68
69#endif
diff --git a/arch/mips/lib/dump_tlb.c b/arch/mips/lib/dump_tlb.c
new file mode 100644
index 000000000..425642363
--- /dev/null
+++ b/arch/mips/lib/dump_tlb.c
@@ -0,0 +1,199 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Dump R4x00 TLB for debugging purposes.
4 *
5 * Copyright (C) 1994, 1995 by Waldorf Electronics, written by Ralf Baechle.
6 * Copyright (C) 1999 by Silicon Graphics, Inc.
7 */
8#include <linux/kernel.h>
9#include <linux/mm.h>
10
11#include <asm/hazards.h>
12#include <asm/mipsregs.h>
13#include <asm/mmu_context.h>
14#include <asm/page.h>
15#include <asm/tlbdebug.h>
16
17void dump_tlb_regs(void)
18{
19 const int field = 2 * sizeof(unsigned long);
20
21 pr_info("Index : %0x\n", read_c0_index());
22 pr_info("PageMask : %0x\n", read_c0_pagemask());
23 if (cpu_has_guestid)
24 pr_info("GuestCtl1: %0x\n", read_c0_guestctl1());
25 pr_info("EntryHi : %0*lx\n", field, read_c0_entryhi());
26 pr_info("EntryLo0 : %0*lx\n", field, read_c0_entrylo0());
27 pr_info("EntryLo1 : %0*lx\n", field, read_c0_entrylo1());
28 pr_info("Wired : %0x\n", read_c0_wired());
29 switch (current_cpu_type()) {
30 case CPU_R10000:
31 case CPU_R12000:
32 case CPU_R14000:
33 case CPU_R16000:
34 pr_info("FrameMask: %0x\n", read_c0_framemask());
35 break;
36 }
37 if (cpu_has_small_pages || cpu_has_rixi || cpu_has_xpa)
38 pr_info("PageGrain: %0x\n", read_c0_pagegrain());
39 if (cpu_has_htw) {
40 pr_info("PWField : %0*lx\n", field, read_c0_pwfield());
41 pr_info("PWSize : %0*lx\n", field, read_c0_pwsize());
42 pr_info("PWCtl : %0x\n", read_c0_pwctl());
43 }
44}
45
46static inline const char *msk2str(unsigned int mask)
47{
48 switch (mask) {
49 case PM_4K: return "4kb";
50 case PM_16K: return "16kb";
51 case PM_64K: return "64kb";
52 case PM_256K: return "256kb";
53#ifdef CONFIG_CPU_CAVIUM_OCTEON
54 case PM_8K: return "8kb";
55 case PM_32K: return "32kb";
56 case PM_128K: return "128kb";
57 case PM_512K: return "512kb";
58 case PM_2M: return "2Mb";
59 case PM_8M: return "8Mb";
60 case PM_32M: return "32Mb";
61#endif
62#ifndef CONFIG_CPU_VR41XX
63 case PM_1M: return "1Mb";
64 case PM_4M: return "4Mb";
65 case PM_16M: return "16Mb";
66 case PM_64M: return "64Mb";
67 case PM_256M: return "256Mb";
68 case PM_1G: return "1Gb";
69#endif
70 }
71 return "";
72}
73
74static void dump_tlb(int first, int last)
75{
76 unsigned long s_entryhi, entryhi, asid, mmid;
77 unsigned long long entrylo0, entrylo1, pa;
78 unsigned int s_index, s_pagemask, s_guestctl1 = 0;
79 unsigned int pagemask, guestctl1 = 0, c0, c1, i;
80 unsigned long asidmask = cpu_asid_mask(&current_cpu_data);
81 int asidwidth = DIV_ROUND_UP(ilog2(asidmask) + 1, 4);
82 unsigned long s_mmid;
83#ifdef CONFIG_32BIT
84 bool xpa = cpu_has_xpa && (read_c0_pagegrain() & PG_ELPA);
85 int pwidth = xpa ? 11 : 8;
86 int vwidth = 8;
87#else
88 bool xpa = false;
89 int pwidth = 11;
90 int vwidth = 11;
91#endif
92
93 s_pagemask = read_c0_pagemask();
94 s_entryhi = read_c0_entryhi();
95 s_index = read_c0_index();
96
97 if (cpu_has_mmid)
98 asid = s_mmid = read_c0_memorymapid();
99 else
100 asid = s_entryhi & asidmask;
101
102 if (cpu_has_guestid)
103 s_guestctl1 = read_c0_guestctl1();
104
105 for (i = first; i <= last; i++) {
106 write_c0_index(i);
107 mtc0_tlbr_hazard();
108 tlb_read();
109 tlb_read_hazard();
110 pagemask = read_c0_pagemask();
111 entryhi = read_c0_entryhi();
112 entrylo0 = read_c0_entrylo0();
113 entrylo1 = read_c0_entrylo1();
114
115 if (cpu_has_mmid)
116 mmid = read_c0_memorymapid();
117 else
118 mmid = entryhi & asidmask;
119
120 if (cpu_has_guestid)
121 guestctl1 = read_c0_guestctl1();
122
123 /* EHINV bit marks entire entry as invalid */
124 if (cpu_has_tlbinv && entryhi & MIPS_ENTRYHI_EHINV)
125 continue;
126 /*
127 * Prior to tlbinv, unused entries have a virtual address of
128 * CKSEG0.
129 */
130 if ((entryhi & ~0x1ffffUL) == CKSEG0)
131 continue;
132 /*
133 * ASID takes effect in absence of G (global) bit.
134 * We check both G bits, even though architecturally they should
135 * match one another, because some revisions of the SB1 core may
136 * leave only a single G bit set after a machine check exception
137 * due to duplicate TLB entry.
138 */
139 if (!((entrylo0 | entrylo1) & ENTRYLO_G) && (mmid != asid))
140 continue;
141
142 /*
143 * Only print entries in use
144 */
145 printk("Index: %2d pgmask=%s ", i, msk2str(pagemask));
146
147 c0 = (entrylo0 & ENTRYLO_C) >> ENTRYLO_C_SHIFT;
148 c1 = (entrylo1 & ENTRYLO_C) >> ENTRYLO_C_SHIFT;
149
150 pr_cont("va=%0*lx asid=%0*lx",
151 vwidth, (entryhi & ~0x1fffUL),
152 asidwidth, mmid);
153 if (cpu_has_guestid)
154 pr_cont(" gid=%02lx",
155 (guestctl1 & MIPS_GCTL1_RID)
156 >> MIPS_GCTL1_RID_SHIFT);
157 /* RI/XI are in awkward places, so mask them off separately */
158 pa = entrylo0 & ~(MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI);
159 if (xpa)
160 pa |= (unsigned long long)readx_c0_entrylo0() << 30;
161 pa = (pa << 6) & PAGE_MASK;
162 pr_cont("\n\t[");
163 if (cpu_has_rixi)
164 pr_cont("ri=%d xi=%d ",
165 (entrylo0 & MIPS_ENTRYLO_RI) ? 1 : 0,
166 (entrylo0 & MIPS_ENTRYLO_XI) ? 1 : 0);
167 pr_cont("pa=%0*llx c=%d d=%d v=%d g=%d] [",
168 pwidth, pa, c0,
169 (entrylo0 & ENTRYLO_D) ? 1 : 0,
170 (entrylo0 & ENTRYLO_V) ? 1 : 0,
171 (entrylo0 & ENTRYLO_G) ? 1 : 0);
172 /* RI/XI are in awkward places, so mask them off separately */
173 pa = entrylo1 & ~(MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI);
174 if (xpa)
175 pa |= (unsigned long long)readx_c0_entrylo1() << 30;
176 pa = (pa << 6) & PAGE_MASK;
177 if (cpu_has_rixi)
178 pr_cont("ri=%d xi=%d ",
179 (entrylo1 & MIPS_ENTRYLO_RI) ? 1 : 0,
180 (entrylo1 & MIPS_ENTRYLO_XI) ? 1 : 0);
181 pr_cont("pa=%0*llx c=%d d=%d v=%d g=%d]\n",
182 pwidth, pa, c1,
183 (entrylo1 & ENTRYLO_D) ? 1 : 0,
184 (entrylo1 & ENTRYLO_V) ? 1 : 0,
185 (entrylo1 & ENTRYLO_G) ? 1 : 0);
186 }
187 printk("\n");
188
189 write_c0_entryhi(s_entryhi);
190 write_c0_index(s_index);
191 write_c0_pagemask(s_pagemask);
192 if (cpu_has_guestid)
193 write_c0_guestctl1(s_guestctl1);
194}
195
196void dump_tlb_all(void)
197{
198 dump_tlb(0, current_cpu_data.tlbsize - 1);
199}
diff --git a/arch/mips/lib/iomap-pci.c b/arch/mips/lib/iomap-pci.c
new file mode 100644
index 000000000..210f5a95e
--- /dev/null
+++ b/arch/mips/lib/iomap-pci.c
@@ -0,0 +1,46 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Implement the default iomap interfaces
4 *
5 * (C) Copyright 2004 Linus Torvalds
6 * (C) Copyright 2006 Ralf Baechle <ralf@linux-mips.org>
7 * (C) Copyright 2007 MIPS Technologies, Inc.
8 * written by Ralf Baechle <ralf@linux-mips.org>
9 */
10#include <linux/pci.h>
11#include <linux/export.h>
12#include <asm/io.h>
13
14#ifdef CONFIG_PCI_DRIVERS_LEGACY
15
16void __iomem *__pci_ioport_map(struct pci_dev *dev,
17 unsigned long port, unsigned int nr)
18{
19 struct pci_controller *ctrl = dev->bus->sysdata;
20 unsigned long base = ctrl->io_map_base;
21
22 /* This will eventually become a BUG_ON but for now be gentle */
23 if (unlikely(!ctrl->io_map_base)) {
24 struct pci_bus *bus = dev->bus;
25 char name[8];
26
27 while (bus->parent)
28 bus = bus->parent;
29
30 ctrl->io_map_base = base = mips_io_port_base;
31
32 sprintf(name, "%04x:%02x", pci_domain_nr(bus), bus->number);
33 printk(KERN_WARNING "io_map_base of root PCI bus %s unset. "
34 "Trying to continue but you better\nfix this issue or "
35 "report it to linux-mips@linux-mips.org or your "
36 "vendor.\n", name);
37#ifdef CONFIG_PCI_DOMAINS
38 panic("To avoid data corruption io_map_base MUST be set with "
39 "multiple PCI domains.");
40#endif
41 }
42
43 return (void __iomem *) (ctrl->io_map_base + port);
44}
45
46#endif /* CONFIG_PCI_DRIVERS_LEGACY */
diff --git a/arch/mips/lib/iomap_copy.c b/arch/mips/lib/iomap_copy.c
new file mode 100644
index 000000000..157500a09
--- /dev/null
+++ b/arch/mips/lib/iomap_copy.c
@@ -0,0 +1,29 @@
1// SPDX-License-Identifier: GPL-2.0-only
2
3#include <linux/export.h>
4#include <linux/io.h>
5
6/**
7 * __ioread64_copy - copy data from MMIO space, in 64-bit units
8 * @to: destination (must be 64-bit aligned)
9 * @from: source, in MMIO space (must be 64-bit aligned)
10 * @count: number of 64-bit quantities to copy
11 *
12 * Copy data from MMIO space to kernel space, in units of 32 or 64 bits at a
13 * time. Order of access is not guaranteed, nor is a memory barrier
14 * performed afterwards.
15 */
16void __ioread64_copy(void *to, const void __iomem *from, size_t count)
17{
18#ifdef CONFIG_64BIT
19 u64 *dst = to;
20 const u64 __iomem *src = from;
21 const u64 __iomem *end = src + count;
22
23 while (src < end)
24 *dst++ = __raw_readq(src++);
25#else
26 __ioread32_copy(to, from, count * 2);
27#endif
28}
29EXPORT_SYMBOL_GPL(__ioread64_copy);
diff --git a/arch/mips/lib/libgcc.h b/arch/mips/lib/libgcc.h
new file mode 100644
index 000000000..199a7f962
--- /dev/null
+++ b/arch/mips/lib/libgcc.h
@@ -0,0 +1,43 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __ASM_LIBGCC_H
3#define __ASM_LIBGCC_H
4
5#include <asm/byteorder.h>
6
7typedef int word_type __attribute__ ((mode (__word__)));
8
9#ifdef __BIG_ENDIAN
10struct DWstruct {
11 int high, low;
12};
13
14struct TWstruct {
15 long long high, low;
16};
17#elif defined(__LITTLE_ENDIAN)
18struct DWstruct {
19 int low, high;
20};
21
22struct TWstruct {
23 long long low, high;
24};
25#else
26#error I feel sick.
27#endif
28
29typedef union {
30 struct DWstruct s;
31 long long ll;
32} DWunion;
33
34#if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPSR6)
35typedef int ti_type __attribute__((mode(TI)));
36
37typedef union {
38 struct TWstruct s;
39 ti_type ti;
40} TWunion;
41#endif
42
43#endif /* __ASM_LIBGCC_H */
diff --git a/arch/mips/lib/memcpy.S b/arch/mips/lib/memcpy.S
new file mode 100644
index 000000000..88065ee43
--- /dev/null
+++ b/arch/mips/lib/memcpy.S
@@ -0,0 +1,709 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Unified implementation of memcpy, memmove and the __copy_user backend.
7 *
8 * Copyright (C) 1998, 99, 2000, 01, 2002 Ralf Baechle (ralf@gnu.org)
9 * Copyright (C) 1999, 2000, 01, 2002 Silicon Graphics, Inc.
10 * Copyright (C) 2002 Broadcom, Inc.
11 * memcpy/copy_user author: Mark Vandevoorde
12 * Copyright (C) 2007 Maciej W. Rozycki
13 * Copyright (C) 2014 Imagination Technologies Ltd.
14 *
15 * Mnemonic names for arguments to memcpy/__copy_user
16 */
17
18/*
19 * Hack to resolve longstanding prefetch issue
20 *
21 * Prefetching may be fatal on some systems if we're prefetching beyond the
22 * end of memory on some systems. It's also a seriously bad idea on non
23 * dma-coherent systems.
24 */
25#ifdef CONFIG_DMA_NONCOHERENT
26#undef CONFIG_CPU_HAS_PREFETCH
27#endif
28#ifdef CONFIG_MIPS_MALTA
29#undef CONFIG_CPU_HAS_PREFETCH
30#endif
31#ifdef CONFIG_CPU_MIPSR6
32#undef CONFIG_CPU_HAS_PREFETCH
33#endif
34
35#include <asm/asm.h>
36#include <asm/asm-offsets.h>
37#include <asm/export.h>
38#include <asm/regdef.h>
39
40#define dst a0
41#define src a1
42#define len a2
43
44/*
45 * Spec
46 *
47 * memcpy copies len bytes from src to dst and sets v0 to dst.
48 * It assumes that
49 * - src and dst don't overlap
50 * - src is readable
51 * - dst is writable
52 * memcpy uses the standard calling convention
53 *
54 * __copy_user copies up to len bytes from src to dst and sets a2 (len) to
55 * the number of uncopied bytes due to an exception caused by a read or write.
56 * __copy_user assumes that src and dst don't overlap, and that the call is
57 * implementing one of the following:
58 * copy_to_user
59 * - src is readable (no exceptions when reading src)
60 * copy_from_user
61 * - dst is writable (no exceptions when writing dst)
62 * __copy_user uses a non-standard calling convention; see
63 * include/asm-mips/uaccess.h
64 *
65 * When an exception happens on a load, the handler must
66 # ensure that all of the destination buffer is overwritten to prevent
67 * leaking information to user mode programs.
68 */
69
70/*
71 * Implementation
72 */
73
74/*
75 * The exception handler for loads requires that:
76 * 1- AT contain the address of the byte just past the end of the source
77 * of the copy,
78 * 2- src_entry <= src < AT, and
79 * 3- (dst - src) == (dst_entry - src_entry),
80 * The _entry suffix denotes values when __copy_user was called.
81 *
82 * (1) is set up up by uaccess.h and maintained by not writing AT in copy_user
83 * (2) is met by incrementing src by the number of bytes copied
84 * (3) is met by not doing loads between a pair of increments of dst and src
85 *
86 * The exception handlers for stores adjust len (if necessary) and return.
87 * These handlers do not need to overwrite any data.
88 *
89 * For __rmemcpy and memmove an exception is always a kernel bug, therefore
90 * they're not protected.
91 */
92
93/* Instruction type */
94#define LD_INSN 1
95#define ST_INSN 2
96/* Pretech type */
97#define SRC_PREFETCH 1
98#define DST_PREFETCH 2
99#define LEGACY_MODE 1
100#define EVA_MODE 2
101#define USEROP 1
102#define KERNELOP 2
103
104/*
105 * Wrapper to add an entry in the exception table
106 * in case the insn causes a memory exception.
107 * Arguments:
108 * insn : Load/store instruction
109 * type : Instruction type
110 * reg : Register
111 * addr : Address
112 * handler : Exception handler
113 */
114
115#define EXC(insn, type, reg, addr, handler) \
116 .if \mode == LEGACY_MODE; \
1179: insn reg, addr; \
118 .section __ex_table,"a"; \
119 PTR 9b, handler; \
120 .previous; \
121 /* This is assembled in EVA mode */ \
122 .else; \
123 /* If loading from user or storing to user */ \
124 .if ((\from == USEROP) && (type == LD_INSN)) || \
125 ((\to == USEROP) && (type == ST_INSN)); \
1269: __BUILD_EVA_INSN(insn##e, reg, addr); \
127 .section __ex_table,"a"; \
128 PTR 9b, handler; \
129 .previous; \
130 .else; \
131 /* \
132 * Still in EVA, but no need for \
133 * exception handler or EVA insn \
134 */ \
135 insn reg, addr; \
136 .endif; \
137 .endif
138
139/*
140 * Only on the 64-bit kernel we can made use of 64-bit registers.
141 */
142#ifdef CONFIG_64BIT
143#define USE_DOUBLE
144#endif
145
146#ifdef USE_DOUBLE
147
148#define LOADK ld /* No exception */
149#define LOAD(reg, addr, handler) EXC(ld, LD_INSN, reg, addr, handler)
150#define LOADL(reg, addr, handler) EXC(ldl, LD_INSN, reg, addr, handler)
151#define LOADR(reg, addr, handler) EXC(ldr, LD_INSN, reg, addr, handler)
152#define STOREL(reg, addr, handler) EXC(sdl, ST_INSN, reg, addr, handler)
153#define STORER(reg, addr, handler) EXC(sdr, ST_INSN, reg, addr, handler)
154#define STORE(reg, addr, handler) EXC(sd, ST_INSN, reg, addr, handler)
155#define ADD daddu
156#define SUB dsubu
157#define SRL dsrl
158#define SRA dsra
159#define SLL dsll
160#define SLLV dsllv
161#define SRLV dsrlv
162#define NBYTES 8
163#define LOG_NBYTES 3
164
165/*
166 * As we are sharing code base with the mips32 tree (which use the o32 ABI
167 * register definitions). We need to redefine the register definitions from
168 * the n64 ABI register naming to the o32 ABI register naming.
169 */
170#undef t0
171#undef t1
172#undef t2
173#undef t3
174#define t0 $8
175#define t1 $9
176#define t2 $10
177#define t3 $11
178#define t4 $12
179#define t5 $13
180#define t6 $14
181#define t7 $15
182
183#else
184
185#define LOADK lw /* No exception */
186#define LOAD(reg, addr, handler) EXC(lw, LD_INSN, reg, addr, handler)
187#define LOADL(reg, addr, handler) EXC(lwl, LD_INSN, reg, addr, handler)
188#define LOADR(reg, addr, handler) EXC(lwr, LD_INSN, reg, addr, handler)
189#define STOREL(reg, addr, handler) EXC(swl, ST_INSN, reg, addr, handler)
190#define STORER(reg, addr, handler) EXC(swr, ST_INSN, reg, addr, handler)
191#define STORE(reg, addr, handler) EXC(sw, ST_INSN, reg, addr, handler)
192#define ADD addu
193#define SUB subu
194#define SRL srl
195#define SLL sll
196#define SRA sra
197#define SLLV sllv
198#define SRLV srlv
199#define NBYTES 4
200#define LOG_NBYTES 2
201
202#endif /* USE_DOUBLE */
203
204#define LOADB(reg, addr, handler) EXC(lb, LD_INSN, reg, addr, handler)
205#define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
206
207#ifdef CONFIG_CPU_HAS_PREFETCH
208# define _PREF(hint, addr, type) \
209 .if \mode == LEGACY_MODE; \
210 kernel_pref(hint, addr); \
211 .else; \
212 .if ((\from == USEROP) && (type == SRC_PREFETCH)) || \
213 ((\to == USEROP) && (type == DST_PREFETCH)); \
214 /* \
215 * PREFE has only 9 bits for the offset \
216 * compared to PREF which has 16, so it may \
217 * need to use the $at register but this \
218 * register should remain intact because it's \
219 * used later on. Therefore use $v1. \
220 */ \
221 .set at=v1; \
222 user_pref(hint, addr); \
223 .set noat; \
224 .else; \
225 kernel_pref(hint, addr); \
226 .endif; \
227 .endif
228#else
229# define _PREF(hint, addr, type)
230#endif
231
232#define PREFS(hint, addr) _PREF(hint, addr, SRC_PREFETCH)
233#define PREFD(hint, addr) _PREF(hint, addr, DST_PREFETCH)
234
235#ifdef CONFIG_CPU_LITTLE_ENDIAN
236#define LDFIRST LOADR
237#define LDREST LOADL
238#define STFIRST STORER
239#define STREST STOREL
240#define SHIFT_DISCARD SLLV
241#else
242#define LDFIRST LOADL
243#define LDREST LOADR
244#define STFIRST STOREL
245#define STREST STORER
246#define SHIFT_DISCARD SRLV
247#endif
248
249#define FIRST(unit) ((unit)*NBYTES)
250#define REST(unit) (FIRST(unit)+NBYTES-1)
251#define UNIT(unit) FIRST(unit)
252
253#define ADDRMASK (NBYTES-1)
254
255 .text
256 .set noreorder
257#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
258 .set noat
259#else
260 .set at=v1
261#endif
262
263 .align 5
264
265 /*
266 * Macro to build the __copy_user common code
267 * Arguments:
268 * mode : LEGACY_MODE or EVA_MODE
269 * from : Source operand. USEROP or KERNELOP
270 * to : Destination operand. USEROP or KERNELOP
271 */
272 .macro __BUILD_COPY_USER mode, from, to
273
274 /* initialize __memcpy if this the first time we execute this macro */
275 .ifnotdef __memcpy
276 .set __memcpy, 1
277 .hidden __memcpy /* make sure it does not leak */
278 .endif
279
280 /*
281 * Note: dst & src may be unaligned, len may be 0
282 * Temps
283 */
284#define rem t8
285
286 R10KCBARRIER(0(ra))
287 /*
288 * The "issue break"s below are very approximate.
289 * Issue delays for dcache fills will perturb the schedule, as will
290 * load queue full replay traps, etc.
291 *
292 * If len < NBYTES use byte operations.
293 */
294 PREFS( 0, 0(src) )
295 PREFD( 1, 0(dst) )
296 sltu t2, len, NBYTES
297 and t1, dst, ADDRMASK
298 PREFS( 0, 1*32(src) )
299 PREFD( 1, 1*32(dst) )
300 bnez t2, .Lcopy_bytes_checklen\@
301 and t0, src, ADDRMASK
302 PREFS( 0, 2*32(src) )
303 PREFD( 1, 2*32(dst) )
304#ifndef CONFIG_CPU_NO_LOAD_STORE_LR
305 bnez t1, .Ldst_unaligned\@
306 nop
307 bnez t0, .Lsrc_unaligned_dst_aligned\@
308#else /* CONFIG_CPU_NO_LOAD_STORE_LR */
309 or t0, t0, t1
310 bnez t0, .Lcopy_unaligned_bytes\@
311#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */
312 /*
313 * use delay slot for fall-through
314 * src and dst are aligned; need to compute rem
315 */
316.Lboth_aligned\@:
317 SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter
318 beqz t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES
319 and rem, len, (8*NBYTES-1) # rem = len % (8*NBYTES)
320 PREFS( 0, 3*32(src) )
321 PREFD( 1, 3*32(dst) )
322 .align 4
3231:
324 R10KCBARRIER(0(ra))
325 LOAD(t0, UNIT(0)(src), .Ll_exc\@)
326 LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
327 LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
328 LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
329 SUB len, len, 8*NBYTES
330 LOAD(t4, UNIT(4)(src), .Ll_exc_copy\@)
331 LOAD(t7, UNIT(5)(src), .Ll_exc_copy\@)
332 STORE(t0, UNIT(0)(dst), .Ls_exc_p8u\@)
333 STORE(t1, UNIT(1)(dst), .Ls_exc_p7u\@)
334 LOAD(t0, UNIT(6)(src), .Ll_exc_copy\@)
335 LOAD(t1, UNIT(7)(src), .Ll_exc_copy\@)
336 ADD src, src, 8*NBYTES
337 ADD dst, dst, 8*NBYTES
338 STORE(t2, UNIT(-6)(dst), .Ls_exc_p6u\@)
339 STORE(t3, UNIT(-5)(dst), .Ls_exc_p5u\@)
340 STORE(t4, UNIT(-4)(dst), .Ls_exc_p4u\@)
341 STORE(t7, UNIT(-3)(dst), .Ls_exc_p3u\@)
342 STORE(t0, UNIT(-2)(dst), .Ls_exc_p2u\@)
343 STORE(t1, UNIT(-1)(dst), .Ls_exc_p1u\@)
344 PREFS( 0, 8*32(src) )
345 PREFD( 1, 8*32(dst) )
346 bne len, rem, 1b
347 nop
348
349 /*
350 * len == rem == the number of bytes left to copy < 8*NBYTES
351 */
352.Lcleanup_both_aligned\@:
353 beqz len, .Ldone\@
354 sltu t0, len, 4*NBYTES
355 bnez t0, .Lless_than_4units\@
356 and rem, len, (NBYTES-1) # rem = len % NBYTES
357 /*
358 * len >= 4*NBYTES
359 */
360 LOAD( t0, UNIT(0)(src), .Ll_exc\@)
361 LOAD( t1, UNIT(1)(src), .Ll_exc_copy\@)
362 LOAD( t2, UNIT(2)(src), .Ll_exc_copy\@)
363 LOAD( t3, UNIT(3)(src), .Ll_exc_copy\@)
364 SUB len, len, 4*NBYTES
365 ADD src, src, 4*NBYTES
366 R10KCBARRIER(0(ra))
367 STORE(t0, UNIT(0)(dst), .Ls_exc_p4u\@)
368 STORE(t1, UNIT(1)(dst), .Ls_exc_p3u\@)
369 STORE(t2, UNIT(2)(dst), .Ls_exc_p2u\@)
370 STORE(t3, UNIT(3)(dst), .Ls_exc_p1u\@)
371 .set reorder /* DADDI_WAR */
372 ADD dst, dst, 4*NBYTES
373 beqz len, .Ldone\@
374 .set noreorder
375.Lless_than_4units\@:
376 /*
377 * rem = len % NBYTES
378 */
379 beq rem, len, .Lcopy_bytes\@
380 nop
3811:
382 R10KCBARRIER(0(ra))
383 LOAD(t0, 0(src), .Ll_exc\@)
384 ADD src, src, NBYTES
385 SUB len, len, NBYTES
386 STORE(t0, 0(dst), .Ls_exc_p1u\@)
387 .set reorder /* DADDI_WAR */
388 ADD dst, dst, NBYTES
389 bne rem, len, 1b
390 .set noreorder
391
392#ifndef CONFIG_CPU_NO_LOAD_STORE_LR
393 /*
394 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
395 * A loop would do only a byte at a time with possible branch
396 * mispredicts. Can't do an explicit LOAD dst,mask,or,STORE
397 * because can't assume read-access to dst. Instead, use
398 * STREST dst, which doesn't require read access to dst.
399 *
400 * This code should perform better than a simple loop on modern,
401 * wide-issue mips processors because the code has fewer branches and
402 * more instruction-level parallelism.
403 */
404#define bits t2
405 beqz len, .Ldone\@
406 ADD t1, dst, len # t1 is just past last byte of dst
407 li bits, 8*NBYTES
408 SLL rem, len, 3 # rem = number of bits to keep
409 LOAD(t0, 0(src), .Ll_exc\@)
410 SUB bits, bits, rem # bits = number of bits to discard
411 SHIFT_DISCARD t0, t0, bits
412 STREST(t0, -1(t1), .Ls_exc\@)
413 jr ra
414 move len, zero
415.Ldst_unaligned\@:
416 /*
417 * dst is unaligned
418 * t0 = src & ADDRMASK
419 * t1 = dst & ADDRMASK; T1 > 0
420 * len >= NBYTES
421 *
422 * Copy enough bytes to align dst
423 * Set match = (src and dst have same alignment)
424 */
425#define match rem
426 LDFIRST(t3, FIRST(0)(src), .Ll_exc\@)
427 ADD t2, zero, NBYTES
428 LDREST(t3, REST(0)(src), .Ll_exc_copy\@)
429 SUB t2, t2, t1 # t2 = number of bytes copied
430 xor match, t0, t1
431 R10KCBARRIER(0(ra))
432 STFIRST(t3, FIRST(0)(dst), .Ls_exc\@)
433 beq len, t2, .Ldone\@
434 SUB len, len, t2
435 ADD dst, dst, t2
436 beqz match, .Lboth_aligned\@
437 ADD src, src, t2
438
439.Lsrc_unaligned_dst_aligned\@:
440 SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter
441 PREFS( 0, 3*32(src) )
442 beqz t0, .Lcleanup_src_unaligned\@
443 and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES
444 PREFD( 1, 3*32(dst) )
4451:
446/*
447 * Avoid consecutive LD*'s to the same register since some mips
448 * implementations can't issue them in the same cycle.
449 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
450 * are to the same unit (unless src is aligned, but it's not).
451 */
452 R10KCBARRIER(0(ra))
453 LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
454 LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy\@)
455 SUB len, len, 4*NBYTES
456 LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
457 LDREST(t1, REST(1)(src), .Ll_exc_copy\@)
458 LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy\@)
459 LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy\@)
460 LDREST(t2, REST(2)(src), .Ll_exc_copy\@)
461 LDREST(t3, REST(3)(src), .Ll_exc_copy\@)
462 PREFS( 0, 9*32(src) ) # 0 is PREF_LOAD (not streamed)
463 ADD src, src, 4*NBYTES
464#ifdef CONFIG_CPU_SB1
465 nop # improves slotting
466#endif
467 STORE(t0, UNIT(0)(dst), .Ls_exc_p4u\@)
468 STORE(t1, UNIT(1)(dst), .Ls_exc_p3u\@)
469 STORE(t2, UNIT(2)(dst), .Ls_exc_p2u\@)
470 STORE(t3, UNIT(3)(dst), .Ls_exc_p1u\@)
471 PREFD( 1, 9*32(dst) ) # 1 is PREF_STORE (not streamed)
472 .set reorder /* DADDI_WAR */
473 ADD dst, dst, 4*NBYTES
474 bne len, rem, 1b
475 .set noreorder
476
477.Lcleanup_src_unaligned\@:
478 beqz len, .Ldone\@
479 and rem, len, NBYTES-1 # rem = len % NBYTES
480 beq rem, len, .Lcopy_bytes\@
481 nop
4821:
483 R10KCBARRIER(0(ra))
484 LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
485 LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
486 ADD src, src, NBYTES
487 SUB len, len, NBYTES
488 STORE(t0, 0(dst), .Ls_exc_p1u\@)
489 .set reorder /* DADDI_WAR */
490 ADD dst, dst, NBYTES
491 bne len, rem, 1b
492 .set noreorder
493
494#endif /* !CONFIG_CPU_NO_LOAD_STORE_LR */
495.Lcopy_bytes_checklen\@:
496 beqz len, .Ldone\@
497 nop
498.Lcopy_bytes\@:
499 /* 0 < len < NBYTES */
500 R10KCBARRIER(0(ra))
501#define COPY_BYTE(N) \
502 LOADB(t0, N(src), .Ll_exc\@); \
503 SUB len, len, 1; \
504 beqz len, .Ldone\@; \
505 STOREB(t0, N(dst), .Ls_exc_p1\@)
506
507 COPY_BYTE(0)
508 COPY_BYTE(1)
509#ifdef USE_DOUBLE
510 COPY_BYTE(2)
511 COPY_BYTE(3)
512 COPY_BYTE(4)
513 COPY_BYTE(5)
514#endif
515 LOADB(t0, NBYTES-2(src), .Ll_exc\@)
516 SUB len, len, 1
517 jr ra
518 STOREB(t0, NBYTES-2(dst), .Ls_exc_p1\@)
519.Ldone\@:
520 jr ra
521 nop
522
523#ifdef CONFIG_CPU_NO_LOAD_STORE_LR
524.Lcopy_unaligned_bytes\@:
5251:
526 COPY_BYTE(0)
527 COPY_BYTE(1)
528 COPY_BYTE(2)
529 COPY_BYTE(3)
530 COPY_BYTE(4)
531 COPY_BYTE(5)
532 COPY_BYTE(6)
533 COPY_BYTE(7)
534 ADD src, src, 8
535 b 1b
536 ADD dst, dst, 8
537#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */
538 .if __memcpy == 1
539 END(memcpy)
540 .set __memcpy, 0
541 .hidden __memcpy
542 .endif
543
544.Ll_exc_copy\@:
545 /*
546 * Copy bytes from src until faulting load address (or until a
547 * lb faults)
548 *
549 * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
550 * may be more than a byte beyond the last address.
551 * Hence, the lb below may get an exception.
552 *
553 * Assumes src < THREAD_BUADDR($28)
554 */
555 LOADK t0, TI_TASK($28)
556 nop
557 LOADK t0, THREAD_BUADDR(t0)
5581:
559 LOADB(t1, 0(src), .Ll_exc\@)
560 ADD src, src, 1
561 sb t1, 0(dst) # can't fault -- we're copy_from_user
562 .set reorder /* DADDI_WAR */
563 ADD dst, dst, 1
564 bne src, t0, 1b
565 .set noreorder
566.Ll_exc\@:
567 LOADK t0, TI_TASK($28)
568 nop
569 LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address
570 nop
571 SUB len, AT, t0 # len number of uncopied bytes
572 jr ra
573 nop
574
575#define SEXC(n) \
576 .set reorder; /* DADDI_WAR */ \
577.Ls_exc_p ## n ## u\@: \
578 ADD len, len, n*NBYTES; \
579 jr ra; \
580 .set noreorder
581
582SEXC(8)
583SEXC(7)
584SEXC(6)
585SEXC(5)
586SEXC(4)
587SEXC(3)
588SEXC(2)
589SEXC(1)
590
591.Ls_exc_p1\@:
592 .set reorder /* DADDI_WAR */
593 ADD len, len, 1
594 jr ra
595 .set noreorder
596.Ls_exc\@:
597 jr ra
598 nop
599 .endm
600
601#ifndef CONFIG_HAVE_PLAT_MEMCPY
602 .align 5
603LEAF(memmove)
604EXPORT_SYMBOL(memmove)
605 ADD t0, a0, a2
606 ADD t1, a1, a2
607 sltu t0, a1, t0 # dst + len <= src -> memcpy
608 sltu t1, a0, t1 # dst >= src + len -> memcpy
609 and t0, t1
610 beqz t0, .L__memcpy
611 move v0, a0 /* return value */
612 beqz a2, .Lr_out
613 END(memmove)
614
615 /* fall through to __rmemcpy */
616LEAF(__rmemcpy) /* a0=dst a1=src a2=len */
617 sltu t0, a1, a0
618 beqz t0, .Lr_end_bytes_up # src >= dst
619 nop
620 ADD a0, a2 # dst = dst + len
621 ADD a1, a2 # src = src + len
622
623.Lr_end_bytes:
624 R10KCBARRIER(0(ra))
625 lb t0, -1(a1)
626 SUB a2, a2, 0x1
627 sb t0, -1(a0)
628 SUB a1, a1, 0x1
629 .set reorder /* DADDI_WAR */
630 SUB a0, a0, 0x1
631 bnez a2, .Lr_end_bytes
632 .set noreorder
633
634.Lr_out:
635 jr ra
636 move a2, zero
637
638.Lr_end_bytes_up:
639 R10KCBARRIER(0(ra))
640 lb t0, (a1)
641 SUB a2, a2, 0x1
642 sb t0, (a0)
643 ADD a1, a1, 0x1
644 .set reorder /* DADDI_WAR */
645 ADD a0, a0, 0x1
646 bnez a2, .Lr_end_bytes_up
647 .set noreorder
648
649 jr ra
650 move a2, zero
651 END(__rmemcpy)
652
653/*
654 * A combined memcpy/__copy_user
655 * __copy_user sets len to 0 for success; else to an upper bound of
656 * the number of uncopied bytes.
657 * memcpy sets v0 to dst.
658 */
659 .align 5
660LEAF(memcpy) /* a0=dst a1=src a2=len */
661EXPORT_SYMBOL(memcpy)
662 move v0, dst /* return value */
663.L__memcpy:
664FEXPORT(__copy_user)
665EXPORT_SYMBOL(__copy_user)
666 /* Legacy Mode, user <-> user */
667 __BUILD_COPY_USER LEGACY_MODE USEROP USEROP
668
669#endif
670
671#ifdef CONFIG_EVA
672
673/*
674 * For EVA we need distinct symbols for reading and writing to user space.
675 * This is because we need to use specific EVA instructions to perform the
676 * virtual <-> physical translation when a virtual address is actually in user
677 * space
678 */
679
680/*
681 * __copy_from_user (EVA)
682 */
683
684LEAF(__copy_from_user_eva)
685EXPORT_SYMBOL(__copy_from_user_eva)
686 __BUILD_COPY_USER EVA_MODE USEROP KERNELOP
687END(__copy_from_user_eva)
688
689
690
691/*
692 * __copy_to_user (EVA)
693 */
694
695LEAF(__copy_to_user_eva)
696EXPORT_SYMBOL(__copy_to_user_eva)
697__BUILD_COPY_USER EVA_MODE KERNELOP USEROP
698END(__copy_to_user_eva)
699
700/*
701 * __copy_in_user (EVA)
702 */
703
704LEAF(__copy_in_user_eva)
705EXPORT_SYMBOL(__copy_in_user_eva)
706__BUILD_COPY_USER EVA_MODE USEROP USEROP
707END(__copy_in_user_eva)
708
709#endif
diff --git a/arch/mips/lib/memset.S b/arch/mips/lib/memset.S
new file mode 100644
index 000000000..d5449e8a3
--- /dev/null
+++ b/arch/mips/lib/memset.S
@@ -0,0 +1,328 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1998, 1999, 2000 by Ralf Baechle
7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8 * Copyright (C) 2007 by Maciej W. Rozycki
9 * Copyright (C) 2011, 2012 MIPS Technologies, Inc.
10 */
11#include <asm/asm.h>
12#include <asm/asm-offsets.h>
13#include <asm/export.h>
14#include <asm/regdef.h>
15
16#if LONGSIZE == 4
17#define LONG_S_L swl
18#define LONG_S_R swr
19#else
20#define LONG_S_L sdl
21#define LONG_S_R sdr
22#endif
23
24#ifdef CONFIG_CPU_MICROMIPS
25#define STORSIZE (LONGSIZE * 2)
26#define STORMASK (STORSIZE - 1)
27#define FILL64RG t8
28#define FILLPTRG t7
29#undef LONG_S
30#define LONG_S LONG_SP
31#else
32#define STORSIZE LONGSIZE
33#define STORMASK LONGMASK
34#define FILL64RG a1
35#define FILLPTRG t0
36#endif
37
38#define LEGACY_MODE 1
39#define EVA_MODE 2
40
41/*
42 * No need to protect it with EVA #ifdefery. The generated block of code
43 * will never be assembled if EVA is not enabled.
44 */
45#define __EVAFY(insn, reg, addr) __BUILD_EVA_INSN(insn##e, reg, addr)
46#define ___BUILD_EVA_INSN(insn, reg, addr) __EVAFY(insn, reg, addr)
47
48#define EX(insn,reg,addr,handler) \
49 .if \mode == LEGACY_MODE; \
509: insn reg, addr; \
51 .else; \
529: ___BUILD_EVA_INSN(insn, reg, addr); \
53 .endif; \
54 .section __ex_table,"a"; \
55 PTR 9b, handler; \
56 .previous
57
58 .macro f_fill64 dst, offset, val, fixup, mode
59 EX(LONG_S, \val, (\offset + 0 * STORSIZE)(\dst), \fixup)
60 EX(LONG_S, \val, (\offset + 1 * STORSIZE)(\dst), \fixup)
61 EX(LONG_S, \val, (\offset + 2 * STORSIZE)(\dst), \fixup)
62 EX(LONG_S, \val, (\offset + 3 * STORSIZE)(\dst), \fixup)
63#if ((defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4)) || !defined(CONFIG_CPU_MICROMIPS))
64 EX(LONG_S, \val, (\offset + 4 * STORSIZE)(\dst), \fixup)
65 EX(LONG_S, \val, (\offset + 5 * STORSIZE)(\dst), \fixup)
66 EX(LONG_S, \val, (\offset + 6 * STORSIZE)(\dst), \fixup)
67 EX(LONG_S, \val, (\offset + 7 * STORSIZE)(\dst), \fixup)
68#endif
69#if (!defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4))
70 EX(LONG_S, \val, (\offset + 8 * STORSIZE)(\dst), \fixup)
71 EX(LONG_S, \val, (\offset + 9 * STORSIZE)(\dst), \fixup)
72 EX(LONG_S, \val, (\offset + 10 * STORSIZE)(\dst), \fixup)
73 EX(LONG_S, \val, (\offset + 11 * STORSIZE)(\dst), \fixup)
74 EX(LONG_S, \val, (\offset + 12 * STORSIZE)(\dst), \fixup)
75 EX(LONG_S, \val, (\offset + 13 * STORSIZE)(\dst), \fixup)
76 EX(LONG_S, \val, (\offset + 14 * STORSIZE)(\dst), \fixup)
77 EX(LONG_S, \val, (\offset + 15 * STORSIZE)(\dst), \fixup)
78#endif
79 .endm
80
81 .align 5
82
83 /*
84 * Macro to generate the __bzero{,_user} symbol
85 * Arguments:
86 * mode: LEGACY_MODE or EVA_MODE
87 */
88 .macro __BUILD_BZERO mode
89 /* Initialize __memset if this is the first time we call this macro */
90 .ifnotdef __memset
91 .set __memset, 1
92 .hidden __memset /* Make sure it does not leak */
93 .endif
94
95 sltiu t0, a2, STORSIZE /* very small region? */
96 .set noreorder
97 bnez t0, .Lsmall_memset\@
98 andi t0, a0, STORMASK /* aligned? */
99 .set reorder
100
101#ifdef CONFIG_CPU_MICROMIPS
102 move t8, a1 /* used by 'swp' instruction */
103 move t9, a1
104#endif
105 .set noreorder
106#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
107 beqz t0, 1f
108 PTR_SUBU t0, STORSIZE /* alignment in bytes */
109#else
110 .set noat
111 li AT, STORSIZE
112 beqz t0, 1f
113 PTR_SUBU t0, AT /* alignment in bytes */
114 .set at
115#endif
116 .set reorder
117
118#ifndef CONFIG_CPU_NO_LOAD_STORE_LR
119 R10KCBARRIER(0(ra))
120#ifdef __MIPSEB__
121 EX(LONG_S_L, a1, (a0), .Lfirst_fixup\@) /* make word/dword aligned */
122#else
123 EX(LONG_S_R, a1, (a0), .Lfirst_fixup\@) /* make word/dword aligned */
124#endif
125 PTR_SUBU a0, t0 /* long align ptr */
126 PTR_ADDU a2, t0 /* correct size */
127
128#else /* CONFIG_CPU_NO_LOAD_STORE_LR */
129#define STORE_BYTE(N) \
130 EX(sb, a1, N(a0), .Lbyte_fixup\@); \
131 .set noreorder; \
132 beqz t0, 0f; \
133 PTR_ADDU t0, 1; \
134 .set reorder;
135
136 PTR_ADDU a2, t0 /* correct size */
137 PTR_ADDU t0, 1
138 STORE_BYTE(0)
139 STORE_BYTE(1)
140#if LONGSIZE == 4
141 EX(sb, a1, 2(a0), .Lbyte_fixup\@)
142#else
143 STORE_BYTE(2)
144 STORE_BYTE(3)
145 STORE_BYTE(4)
146 STORE_BYTE(5)
147 EX(sb, a1, 6(a0), .Lbyte_fixup\@)
148#endif
1490:
150 ori a0, STORMASK
151 xori a0, STORMASK
152 PTR_ADDIU a0, STORSIZE
153#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */
1541: ori t1, a2, 0x3f /* # of full blocks */
155 xori t1, 0x3f
156 andi t0, a2, 0x40-STORSIZE
157 beqz t1, .Lmemset_partial\@ /* no block to fill */
158
159 PTR_ADDU t1, a0 /* end address */
1601: PTR_ADDIU a0, 64
161 R10KCBARRIER(0(ra))
162 f_fill64 a0, -64, FILL64RG, .Lfwd_fixup\@, \mode
163 bne t1, a0, 1b
164
165.Lmemset_partial\@:
166 R10KCBARRIER(0(ra))
167 PTR_LA t1, 2f /* where to start */
168#ifdef CONFIG_CPU_MICROMIPS
169 LONG_SRL t7, t0, 1
170#endif
171#if LONGSIZE == 4
172 PTR_SUBU t1, FILLPTRG
173#else
174 .set noat
175 LONG_SRL AT, FILLPTRG, 1
176 PTR_SUBU t1, AT
177 .set at
178#endif
179 PTR_ADDU a0, t0 /* dest ptr */
180 jr t1
181
182 /* ... but first do longs ... */
183 f_fill64 a0, -64, FILL64RG, .Lpartial_fixup\@, \mode
1842: andi a2, STORMASK /* At most one long to go */
185
186 .set noreorder
187 beqz a2, 1f
188#ifndef CONFIG_CPU_NO_LOAD_STORE_LR
189 PTR_ADDU a0, a2 /* What's left */
190 .set reorder
191 R10KCBARRIER(0(ra))
192#ifdef __MIPSEB__
193 EX(LONG_S_R, a1, -1(a0), .Llast_fixup\@)
194#else
195 EX(LONG_S_L, a1, -1(a0), .Llast_fixup\@)
196#endif
197#else /* CONFIG_CPU_NO_LOAD_STORE_LR */
198 PTR_SUBU t0, $0, a2
199 .set reorder
200 move a2, zero /* No remaining longs */
201 PTR_ADDIU t0, 1
202 STORE_BYTE(0)
203 STORE_BYTE(1)
204#if LONGSIZE == 4
205 EX(sb, a1, 2(a0), .Lbyte_fixup\@)
206#else
207 STORE_BYTE(2)
208 STORE_BYTE(3)
209 STORE_BYTE(4)
210 STORE_BYTE(5)
211 EX(sb, a1, 6(a0), .Lbyte_fixup\@)
212#endif
2130:
214#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */
2151: move a2, zero
216 jr ra
217
218.Lsmall_memset\@:
219 PTR_ADDU t1, a0, a2
220 beqz a2, 2f
221
2221: PTR_ADDIU a0, 1 /* fill bytewise */
223 R10KCBARRIER(0(ra))
224 .set noreorder
225 bne t1, a0, 1b
226 EX(sb, a1, -1(a0), .Lsmall_fixup\@)
227 .set reorder
228
2292: move a2, zero
230 jr ra /* done */
231 .if __memset == 1
232 END(memset)
233 .set __memset, 0
234 .hidden __memset
235 .endif
236
237#ifdef CONFIG_CPU_NO_LOAD_STORE_LR
238.Lbyte_fixup\@:
239 /*
240 * unset_bytes = (#bytes - (#unaligned bytes)) - (-#unaligned bytes remaining + 1) + 1
241 * a2 = a2 - t0 + 1
242 */
243 PTR_SUBU a2, t0
244 PTR_ADDIU a2, 1
245 jr ra
246#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */
247
248.Lfirst_fixup\@:
249 /* unset_bytes already in a2 */
250 jr ra
251
252.Lfwd_fixup\@:
253 /*
254 * unset_bytes = partial_start_addr + #bytes - fault_addr
255 * a2 = t1 + (a2 & 3f) - $28->task->BUADDR
256 */
257 PTR_L t0, TI_TASK($28)
258 andi a2, 0x3f
259 LONG_L t0, THREAD_BUADDR(t0)
260 LONG_ADDU a2, t1
261 LONG_SUBU a2, t0
262 jr ra
263
264.Lpartial_fixup\@:
265 /*
266 * unset_bytes = partial_end_addr + #bytes - fault_addr
267 * a2 = a0 + (a2 & STORMASK) - $28->task->BUADDR
268 */
269 PTR_L t0, TI_TASK($28)
270 andi a2, STORMASK
271 LONG_L t0, THREAD_BUADDR(t0)
272 LONG_ADDU a2, a0
273 LONG_SUBU a2, t0
274 jr ra
275
276.Llast_fixup\@:
277 /* unset_bytes already in a2 */
278 jr ra
279
280.Lsmall_fixup\@:
281 /*
282 * unset_bytes = end_addr - current_addr + 1
283 * a2 = t1 - a0 + 1
284 */
285 PTR_SUBU a2, t1, a0
286 PTR_ADDIU a2, 1
287 jr ra
288
289 .endm
290
291/*
292 * memset(void *s, int c, size_t n)
293 *
294 * a0: start of area to clear
295 * a1: char to fill with
296 * a2: size of area to clear
297 */
298
299LEAF(memset)
300EXPORT_SYMBOL(memset)
301 move v0, a0 /* result */
302 beqz a1, 1f
303
304 andi a1, 0xff /* spread fillword */
305 LONG_SLL t1, a1, 8
306 or a1, t1
307 LONG_SLL t1, a1, 16
308#if LONGSIZE == 8
309 or a1, t1
310 LONG_SLL t1, a1, 32
311#endif
312 or a1, t1
3131:
314#ifndef CONFIG_EVA
315FEXPORT(__bzero)
316EXPORT_SYMBOL(__bzero)
317#else
318FEXPORT(__bzero_kernel)
319EXPORT_SYMBOL(__bzero_kernel)
320#endif
321 __BUILD_BZERO LEGACY_MODE
322
323#ifdef CONFIG_EVA
324LEAF(__bzero)
325EXPORT_SYMBOL(__bzero)
326 __BUILD_BZERO EVA_MODE
327END(__bzero)
328#endif
diff --git a/arch/mips/lib/mips-atomic.c b/arch/mips/lib/mips-atomic.c
new file mode 100644
index 000000000..a9b72eacf
--- /dev/null
+++ b/arch/mips/lib/mips-atomic.c
@@ -0,0 +1,113 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1994, 95, 96, 97, 98, 99, 2003 by Ralf Baechle
7 * Copyright (C) 1996 by Paul M. Antoine
8 * Copyright (C) 1999 Silicon Graphics
9 * Copyright (C) 2000 MIPS Technologies, Inc.
10 */
11#include <asm/irqflags.h>
12#include <asm/hazards.h>
13#include <linux/compiler.h>
14#include <linux/preempt.h>
15#include <linux/export.h>
16#include <linux/stringify.h>
17
18#if !defined(CONFIG_CPU_HAS_DIEI)
19
20/*
21 * For cli() we have to insert nops to make sure that the new value
22 * has actually arrived in the status register before the end of this
23 * macro.
24 * R4000/R4400 need three nops, the R4600 two nops and the R10000 needs
25 * no nops at all.
26 */
27/*
28 * For TX49, operating only IE bit is not enough.
29 *
30 * If mfc0 $12 follows store and the mfc0 is last instruction of a
31 * page and fetching the next instruction causes TLB miss, the result
32 * of the mfc0 might wrongly contain EXL bit.
33 *
34 * ERT-TX49H2-027, ERT-TX49H3-012, ERT-TX49HL3-006, ERT-TX49H4-008
35 *
36 * Workaround: mask EXL bit of the result or place a nop before mfc0.
37 */
38notrace void arch_local_irq_disable(void)
39{
40 preempt_disable_notrace();
41
42 __asm__ __volatile__(
43 " .set push \n"
44 " .set noat \n"
45 " mfc0 $1,$12 \n"
46 " ori $1,0x1f \n"
47 " xori $1,0x1f \n"
48 " .set noreorder \n"
49 " mtc0 $1,$12 \n"
50 " " __stringify(__irq_disable_hazard) " \n"
51 " .set pop \n"
52 : /* no outputs */
53 : /* no inputs */
54 : "memory");
55
56 preempt_enable_notrace();
57}
58EXPORT_SYMBOL(arch_local_irq_disable);
59
60notrace unsigned long arch_local_irq_save(void)
61{
62 unsigned long flags;
63
64 preempt_disable_notrace();
65
66 __asm__ __volatile__(
67 " .set push \n"
68 " .set reorder \n"
69 " .set noat \n"
70 " mfc0 %[flags], $12 \n"
71 " ori $1, %[flags], 0x1f \n"
72 " xori $1, 0x1f \n"
73 " .set noreorder \n"
74 " mtc0 $1, $12 \n"
75 " " __stringify(__irq_disable_hazard) " \n"
76 " .set pop \n"
77 : [flags] "=r" (flags)
78 : /* no inputs */
79 : "memory");
80
81 preempt_enable_notrace();
82
83 return flags;
84}
85EXPORT_SYMBOL(arch_local_irq_save);
86
87notrace void arch_local_irq_restore(unsigned long flags)
88{
89 unsigned long __tmp1;
90
91 preempt_disable_notrace();
92
93 __asm__ __volatile__(
94 " .set push \n"
95 " .set noreorder \n"
96 " .set noat \n"
97 " mfc0 $1, $12 \n"
98 " andi %[flags], 1 \n"
99 " ori $1, 0x1f \n"
100 " xori $1, 0x1f \n"
101 " or %[flags], $1 \n"
102 " mtc0 %[flags], $12 \n"
103 " " __stringify(__irq_disable_hazard) " \n"
104 " .set pop \n"
105 : [flags] "=r" (__tmp1)
106 : "0" (flags)
107 : "memory");
108
109 preempt_enable_notrace();
110}
111EXPORT_SYMBOL(arch_local_irq_restore);
112
113#endif /* !CONFIG_CPU_HAS_DIEI */
diff --git a/arch/mips/lib/multi3.c b/arch/mips/lib/multi3.c
new file mode 100644
index 000000000..4c2483f41
--- /dev/null
+++ b/arch/mips/lib/multi3.c
@@ -0,0 +1,54 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/export.h>
3
4#include "libgcc.h"
5
6/*
7 * GCC 7 & older can suboptimally generate __multi3 calls for mips64r6, so for
8 * that specific case only we implement that intrinsic here.
9 *
10 * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82981
11 */
12#if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPSR6) && (__GNUC__ < 8)
13
14/* multiply 64-bit values, low 64-bits returned */
15static inline long long notrace dmulu(long long a, long long b)
16{
17 long long res;
18
19 asm ("dmulu %0,%1,%2" : "=r" (res) : "r" (a), "r" (b));
20 return res;
21}
22
23/* multiply 64-bit unsigned values, high 64-bits of 128-bit result returned */
24static inline long long notrace dmuhu(long long a, long long b)
25{
26 long long res;
27
28 asm ("dmuhu %0,%1,%2" : "=r" (res) : "r" (a), "r" (b));
29 return res;
30}
31
32/* multiply 128-bit values, low 128-bits returned */
33ti_type notrace __multi3(ti_type a, ti_type b)
34{
35 TWunion res, aa, bb;
36
37 aa.ti = a;
38 bb.ti = b;
39
40 /*
41 * a * b = (a.lo * b.lo)
42 * + 2^64 * (a.hi * b.lo + a.lo * b.hi)
43 * [+ 2^128 * (a.hi * b.hi)]
44 */
45 res.s.low = dmulu(aa.s.low, bb.s.low);
46 res.s.high = dmuhu(aa.s.low, bb.s.low);
47 res.s.high += dmulu(aa.s.high, bb.s.low);
48 res.s.high += dmulu(aa.s.low, bb.s.high);
49
50 return res.ti;
51}
52EXPORT_SYMBOL(__multi3);
53
54#endif /* 64BIT && CPU_MIPSR6 && GCC7 */
diff --git a/arch/mips/lib/r3k_dump_tlb.c b/arch/mips/lib/r3k_dump_tlb.c
new file mode 100644
index 000000000..10b4bf7f7
--- /dev/null
+++ b/arch/mips/lib/r3k_dump_tlb.c
@@ -0,0 +1,75 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Dump R3000 TLB for debugging purposes.
4 *
5 * Copyright (C) 1994, 1995 by Waldorf Electronics, written by Ralf Baechle.
6 * Copyright (C) 1999 by Silicon Graphics, Inc.
7 * Copyright (C) 1999 by Harald Koerfgen
8 */
9#include <linux/kernel.h>
10#include <linux/mm.h>
11
12#include <asm/mipsregs.h>
13#include <asm/mmu_context.h>
14#include <asm/page.h>
15#include <asm/tlbdebug.h>
16
17extern int r3k_have_wired_reg;
18
19void dump_tlb_regs(void)
20{
21 pr_info("Index : %0x\n", read_c0_index());
22 pr_info("EntryHi : %0lx\n", read_c0_entryhi());
23 pr_info("EntryLo : %0lx\n", read_c0_entrylo0());
24 if (r3k_have_wired_reg)
25 pr_info("Wired : %0x\n", read_c0_wired());
26}
27
28static void dump_tlb(int first, int last)
29{
30 int i;
31 unsigned int asid;
32 unsigned long entryhi, entrylo0, asid_mask;
33
34 asid_mask = cpu_asid_mask(&current_cpu_data);
35 asid = read_c0_entryhi() & asid_mask;
36
37 for (i = first; i <= last; i++) {
38 write_c0_index(i<<8);
39 __asm__ __volatile__(
40 ".set\tnoreorder\n\t"
41 "tlbr\n\t"
42 "nop\n\t"
43 ".set\treorder");
44 entryhi = read_c0_entryhi();
45 entrylo0 = read_c0_entrylo0();
46
47 /* Unused entries have a virtual address of KSEG0. */
48 if ((entryhi & PAGE_MASK) != KSEG0 &&
49 (entrylo0 & R3K_ENTRYLO_G ||
50 (entryhi & asid_mask) == asid)) {
51 /*
52 * Only print entries in use
53 */
54 printk("Index: %2d ", i);
55
56 pr_cont("va=%08lx asid=%08lx"
57 " [pa=%06lx n=%d d=%d v=%d g=%d]",
58 entryhi & PAGE_MASK,
59 entryhi & asid_mask,
60 entrylo0 & PAGE_MASK,
61 (entrylo0 & R3K_ENTRYLO_N) ? 1 : 0,
62 (entrylo0 & R3K_ENTRYLO_D) ? 1 : 0,
63 (entrylo0 & R3K_ENTRYLO_V) ? 1 : 0,
64 (entrylo0 & R3K_ENTRYLO_G) ? 1 : 0);
65 }
66 }
67 printk("\n");
68
69 write_c0_entryhi(asid);
70}
71
72void dump_tlb_all(void)
73{
74 dump_tlb(0, current_cpu_data.tlbsize - 1);
75}
diff --git a/arch/mips/lib/strncpy_user.S b/arch/mips/lib/strncpy_user.S
new file mode 100644
index 000000000..acdff66bd
--- /dev/null
+++ b/arch/mips/lib/strncpy_user.S
@@ -0,0 +1,85 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1996, 1999 by Ralf Baechle
7 * Copyright (C) 2011 MIPS Technologies, Inc.
8 */
9#include <linux/errno.h>
10#include <asm/asm.h>
11#include <asm/asm-offsets.h>
12#include <asm/export.h>
13#include <asm/regdef.h>
14
15#define EX(insn,reg,addr,handler) \
169: insn reg, addr; \
17 .section __ex_table,"a"; \
18 PTR 9b, handler; \
19 .previous
20
21/*
22 * Returns: -EFAULT if exception before terminator, N if the entire
23 * buffer filled, else strlen.
24 */
25
26/*
27 * Ugly special case have to check: we might get passed a user space
28 * pointer which wraps into the kernel space. We don't deal with that. If
29 * it happens at most some bytes of the exceptions handlers will be copied.
30 */
31
32 .macro __BUILD_STRNCPY_ASM func
33LEAF(__strncpy_from_\func\()_asm)
34 LONG_L v0, TI_ADDR_LIMIT($28) # pointer ok?
35 and v0, a1
36 bnez v0, .Lfault\@
37
38 move t0, zero
39 move v1, a1
40.ifeqs "\func","kernel"
411: EX(lbu, v0, (v1), .Lfault\@)
42.else
431: EX(lbue, v0, (v1), .Lfault\@)
44.endif
45 PTR_ADDIU v1, 1
46 R10KCBARRIER(0(ra))
47 sb v0, (a0)
48 beqz v0, 2f
49 PTR_ADDIU t0, 1
50 PTR_ADDIU a0, 1
51 bne t0, a2, 1b
522: PTR_ADDU v0, a1, t0
53 xor v0, a1
54 bltz v0, .Lfault\@
55 move v0, t0
56 jr ra # return n
57 END(__strncpy_from_\func\()_asm)
58
59.Lfault\@:
60 li v0, -EFAULT
61 jr ra
62
63 .section __ex_table,"a"
64 PTR 1b, .Lfault\@
65 .previous
66
67 .endm
68
69#ifndef CONFIG_EVA
70 /* Set aliases */
71 .global __strncpy_from_user_asm
72 .set __strncpy_from_user_asm, __strncpy_from_kernel_asm
73EXPORT_SYMBOL(__strncpy_from_user_asm)
74#endif
75
76__BUILD_STRNCPY_ASM kernel
77EXPORT_SYMBOL(__strncpy_from_kernel_asm)
78
79#ifdef CONFIG_EVA
80 .set push
81 .set eva
82__BUILD_STRNCPY_ASM user
83 .set pop
84EXPORT_SYMBOL(__strncpy_from_user_asm)
85#endif
diff --git a/arch/mips/lib/strnlen_user.S b/arch/mips/lib/strnlen_user.S
new file mode 100644
index 000000000..e1bacf5a3
--- /dev/null
+++ b/arch/mips/lib/strnlen_user.S
@@ -0,0 +1,84 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 1996, 1998, 1999, 2004 by Ralf Baechle
7 * Copyright (c) 1999 Silicon Graphics, Inc.
8 */
9#include <asm/asm.h>
10#include <asm/asm-offsets.h>
11#include <asm/export.h>
12#include <asm/regdef.h>
13
14#define EX(insn,reg,addr,handler) \
159: insn reg, addr; \
16 .section __ex_table,"a"; \
17 PTR 9b, handler; \
18 .previous
19
20/*
21 * Return the size of a string including the ending NUL character up to a
22 * maximum of a1 or 0 in case of error.
23 *
24 * Note: for performance reasons we deliberately accept that a user may
25 * make strlen_user and strnlen_user access the first few KSEG0
26 * bytes. There's nothing secret there. On 64-bit accessing beyond
27 * the maximum is a tad hairier ...
28 */
29 .macro __BUILD_STRNLEN_ASM func
30LEAF(__strnlen_\func\()_asm)
31 LONG_L v0, TI_ADDR_LIMIT($28) # pointer ok?
32 and v0, a0
33 bnez v0, .Lfault\@
34
35 move v0, a0
36 PTR_ADDU a1, a0 # stop pointer
371:
38#ifdef CONFIG_CPU_DADDI_WORKAROUNDS
39 .set noat
40 li AT, 1
41#endif
42 beq v0, a1, 1f # limit reached?
43.ifeqs "\func", "kernel"
44 EX(lb, t0, (v0), .Lfault\@)
45.else
46 EX(lbe, t0, (v0), .Lfault\@)
47.endif
48 .set noreorder
49 bnez t0, 1b
501:
51#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
52 PTR_ADDIU v0, 1
53#else
54 PTR_ADDU v0, AT
55 .set at
56#endif
57 .set reorder
58 PTR_SUBU v0, a0
59 jr ra
60 END(__strnlen_\func\()_asm)
61
62.Lfault\@:
63 move v0, zero
64 jr ra
65 .endm
66
67#ifndef CONFIG_EVA
68 /* Set aliases */
69 .global __strnlen_user_asm
70 .set __strnlen_user_asm, __strnlen_kernel_asm
71EXPORT_SYMBOL(__strnlen_user_asm)
72#endif
73
74__BUILD_STRNLEN_ASM kernel
75EXPORT_SYMBOL(__strnlen_kernel_asm)
76
77#ifdef CONFIG_EVA
78
79 .set push
80 .set eva
81__BUILD_STRNLEN_ASM user
82 .set pop
83EXPORT_SYMBOL(__strnlen_user_asm)
84#endif
diff --git a/arch/mips/lib/uncached.c b/arch/mips/lib/uncached.c
new file mode 100644
index 000000000..f80a67c09
--- /dev/null
+++ b/arch/mips/lib/uncached.c
@@ -0,0 +1,82 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2005 Thiemo Seufer
7 * Copyright (C) 2005 MIPS Technologies, Inc. All rights reserved.
8 * Author: Maciej W. Rozycki <macro@mips.com>
9 */
10
11
12#include <asm/addrspace.h>
13#include <asm/bug.h>
14#include <asm/cacheflush.h>
15
16#ifndef CKSEG2
17#define CKSEG2 CKSSEG
18#endif
19#ifndef TO_PHYS_MASK
20#define TO_PHYS_MASK -1
21#endif
22
23/*
24 * FUNC is executed in one of the uncached segments, depending on its
25 * original address as follows:
26 *
27 * 1. If the original address is in CKSEG0 or CKSEG1, then the uncached
28 * segment used is CKSEG1.
29 * 2. If the original address is in XKPHYS, then the uncached segment
30 * used is XKPHYS(2).
31 * 3. Otherwise it's a bug.
32 *
33 * The same remapping is done with the stack pointer. Stack handling
34 * works because we don't handle stack arguments or more complex return
35 * values, so we can avoid sharing the same stack area between a cached
36 * and the uncached mode.
37 */
38unsigned long run_uncached(void *func)
39{
40 register long ret __asm__("$2");
41 long lfunc = (long)func, ufunc;
42 long usp;
43 long sp;
44
45 __asm__("move %0, $sp" : "=r" (sp));
46
47 if (sp >= (long)CKSEG0 && sp < (long)CKSEG2)
48 usp = CKSEG1ADDR(sp);
49#ifdef CONFIG_64BIT
50 else if ((long long)sp >= (long long)PHYS_TO_XKPHYS(0, 0) &&
51 (long long)sp < (long long)PHYS_TO_XKPHYS(8, 0))
52 usp = PHYS_TO_XKPHYS(K_CALG_UNCACHED,
53 XKPHYS_TO_PHYS((long long)sp));
54#endif
55 else {
56 BUG();
57 usp = sp;
58 }
59 if (lfunc >= (long)CKSEG0 && lfunc < (long)CKSEG2)
60 ufunc = CKSEG1ADDR(lfunc);
61#ifdef CONFIG_64BIT
62 else if ((long long)lfunc >= (long long)PHYS_TO_XKPHYS(0, 0) &&
63 (long long)lfunc < (long long)PHYS_TO_XKPHYS(8, 0))
64 ufunc = PHYS_TO_XKPHYS(K_CALG_UNCACHED,
65 XKPHYS_TO_PHYS((long long)lfunc));
66#endif
67 else {
68 BUG();
69 ufunc = lfunc;
70 }
71
72 __asm__ __volatile__ (
73 " move $16, $sp\n"
74 " move $sp, %1\n"
75 " jalr %2\n"
76 " move $sp, $16"
77 : "=r" (ret)
78 : "r" (usp), "r" (ufunc)
79 : "$16", "$31");
80
81 return ret;
82}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-04-16 09:24:29 +0800