context.c 7.74 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * Based on arch/arm/mm/context.c
 *
 * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved.
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

20
#include <linux/bitops.h>
21
#include <linux/sched.h>
22
#include <linux/slab.h>
23 24
#include <linux/mm.h>

25
#include <asm/cpufeature.h>
26
#include <asm/mmu_context.h>
27
#include <asm/smp.h>
28 29
#include <asm/tlbflush.h>

30 31
static u32 asid_bits;
static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
32

33 34
static atomic64_t asid_generation;
static unsigned long *asid_map;
35

36 37 38
static DEFINE_PER_CPU(atomic64_t, active_asids);
static DEFINE_PER_CPU(u64, reserved_asids);
static cpumask_t tlb_flush_pending;
39

40 41
#define ASID_MASK		(~GENMASK(asid_bits - 1, 0))
#define ASID_FIRST_VERSION	(1UL << asid_bits)
42 43 44 45 46 47 48 49 50 51

#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
#define NUM_USER_ASIDS		(ASID_FIRST_VERSION >> 1)
#define asid2idx(asid)		(((asid) & ~ASID_MASK) >> 1)
#define idx2asid(idx)		(((idx) << 1) & ~ASID_MASK)
#else
#define NUM_USER_ASIDS		(ASID_FIRST_VERSION)
#define asid2idx(asid)		((asid) & ~ASID_MASK)
#define idx2asid(idx)		asid2idx(idx)
#endif
52

53 54 55 56
/* Get the ASIDBits supported by the current CPU */
static u32 get_cpu_asid_bits(void)
{
	u32 asid;
57
	int fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR0_EL1),
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
						ID_AA64MMFR0_ASID_SHIFT);

	switch (fld) {
	default:
		pr_warn("CPU%d: Unknown ASID size (%d); assuming 8-bit\n",
					smp_processor_id(),  fld);
		/* Fallthrough */
	case 0:
		asid = 8;
		break;
	case 2:
		asid = 16;
	}

	return asid;
}

75 76 77 78 79 80 81 82 83 84 85 86
/* Check if the current cpu's ASIDBits is compatible with asid_bits */
void verify_cpu_asid_bits(void)
{
	u32 asid = get_cpu_asid_bits();

	if (asid < asid_bits) {
		/*
		 * We cannot decrease the ASID size at runtime, so panic if we support
		 * fewer ASID bits than the boot CPU.
		 */
		pr_crit("CPU%d: smaller ASID size(%u) than boot CPU (%u)\n",
				smp_processor_id(), asid, asid_bits);
87
		cpu_panic_kernel();
88 89 90
	}
}

91
static void flush_context(unsigned int cpu)
92
{
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
	int i;
	u64 asid;

	/* Update the list of reserved ASIDs and the ASID bitmap. */
	bitmap_clear(asid_map, 0, NUM_USER_ASIDS);

	for_each_possible_cpu(i) {
		asid = atomic64_xchg_relaxed(&per_cpu(active_asids, i), 0);
		/*
		 * If this CPU has already been through a
		 * rollover, but hasn't run another task in
		 * the meantime, we must preserve its reserved
		 * ASID, as this is the only trace we have of
		 * the process it is still running.
		 */
		if (asid == 0)
			asid = per_cpu(reserved_asids, i);
110
		__set_bit(asid2idx(asid), asid_map);
111 112 113
		per_cpu(reserved_asids, i) = asid;
	}

114 115 116 117
	/*
	 * Queue a TLB invalidation for each CPU to perform on next
	 * context-switch
	 */
118
	cpumask_setall(&tlb_flush_pending);
119 120
}

121
static bool check_update_reserved_asid(u64 asid, u64 newasid)
122
{
123
	int cpu;
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
	bool hit = false;

	/*
	 * Iterate over the set of reserved ASIDs looking for a match.
	 * If we find one, then we can update our mm to use newasid
	 * (i.e. the same ASID in the current generation) but we can't
	 * exit the loop early, since we need to ensure that all copies
	 * of the old ASID are updated to reflect the mm. Failure to do
	 * so could result in us missing the reserved ASID in a future
	 * generation.
	 */
	for_each_possible_cpu(cpu) {
		if (per_cpu(reserved_asids, cpu) == asid) {
			hit = true;
			per_cpu(reserved_asids, cpu) = newasid;
		}
	}

	return hit;
143 144
}

145
static u64 new_context(struct mm_struct *mm, unsigned int cpu)
146
{
147 148 149
	static u32 cur_idx = 1;
	u64 asid = atomic64_read(&mm->context.id);
	u64 generation = atomic64_read(&asid_generation);
150

151
	if (asid != 0) {
152 153
		u64 newasid = generation | (asid & ~ASID_MASK);

154
		/*
155 156
		 * If our current ASID was active during a rollover, we
		 * can continue to use it and this was just a false alarm.
157
		 */
158 159
		if (check_update_reserved_asid(asid, newasid))
			return newasid;
160 161 162 163 164

		/*
		 * We had a valid ASID in a previous life, so try to re-use
		 * it if possible.
		 */
165
		if (!__test_and_set_bit(asid2idx(asid), asid_map))
166
			return newasid;
167 168 169
	}

	/*
170
	 * Allocate a free ASID. If we can't find one, take a note of the
171 172 173 174
	 * currently active ASIDs and mark the TLBs as requiring flushes.  We
	 * always count from ASID #2 (index 1), as we use ASID #0 when setting
	 * a reserved TTBR0 for the init_mm and we allocate ASIDs in even/odd
	 * pairs.
175
	 */
176 177 178 179 180 181 182 183 184
	asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);
	if (asid != NUM_USER_ASIDS)
		goto set_asid;

	/* We're out of ASIDs, so increment the global generation count */
	generation = atomic64_add_return_relaxed(ASID_FIRST_VERSION,
						 &asid_generation);
	flush_context(cpu);

185
	/* We have more ASIDs than CPUs, so this will always succeed */
186 187 188 189 190
	asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);

set_asid:
	__set_bit(asid, asid_map);
	cur_idx = asid;
191
	return idx2asid(asid) | generation;
192 193
}

194
void check_and_switch_context(struct mm_struct *mm, unsigned int cpu)
195
{
196
	unsigned long flags;
197
	u64 asid, old_active_asid;
198 199

	asid = atomic64_read(&mm->context.id);
200

201
	/*
202
	 * The memory ordering here is subtle.
203 204 205
	 * If our active_asids is non-zero and the ASID matches the current
	 * generation, then we update the active_asids entry with a relaxed
	 * cmpxchg. Racing with a concurrent rollover means that either:
206
	 *
207
	 * - We get a zero back from the cmpxchg and end up waiting on the
208 209 210
	 *   lock. Taking the lock synchronises with the rollover and so
	 *   we are forced to see the updated generation.
	 *
211
	 * - We get a valid ASID back from the cmpxchg, which means the
212 213
	 *   relaxed xchg in flush_context will treat us as reserved
	 *   because atomic RmWs are totally ordered for a given location.
214
	 */
215 216 217 218 219
	old_active_asid = atomic64_read(&per_cpu(active_asids, cpu));
	if (old_active_asid &&
	    !((asid ^ atomic64_read(&asid_generation)) >> asid_bits) &&
	    atomic64_cmpxchg_relaxed(&per_cpu(active_asids, cpu),
				     old_active_asid, asid))
220 221 222 223 224 225 226 227 228
		goto switch_mm_fastpath;

	raw_spin_lock_irqsave(&cpu_asid_lock, flags);
	/* Check that our ASID belongs to the current generation. */
	asid = atomic64_read(&mm->context.id);
	if ((asid ^ atomic64_read(&asid_generation)) >> asid_bits) {
		asid = new_context(mm, cpu);
		atomic64_set(&mm->context.id, asid);
	}
229

230 231
	if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending))
		local_flush_tlb_all();
232

233 234
	atomic64_set(&per_cpu(active_asids, cpu), asid);
	raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
235

236
switch_mm_fastpath:
237 238 239

	arm64_apply_bp_hardening();

240 241 242 243 244 245
	/*
	 * Defer TTBR0_EL1 setting for user threads to uaccess_enable() when
	 * emulating PAN.
	 */
	if (!system_uses_ttbr0_pan())
		cpu_switch_mm(mm->pgd, mm);
246 247
}

248 249 250 251 252 253 254 255 256
/* Errata workaround post TTBRx_EL1 update. */
asmlinkage void post_ttbr_update_workaround(void)
{
	asm(ALTERNATIVE("nop; nop; nop",
			"ic iallu; dsb nsh; isb",
			ARM64_WORKAROUND_CAVIUM_27456,
			CONFIG_CAVIUM_ERRATUM_27456));
}

257
static int asids_init(void)
258
{
259
	asid_bits = get_cpu_asid_bits();
260 261 262 263 264
	/*
	 * Expect allocation after rollover to fail if we don't have at least
	 * one more ASID than CPUs. ASID #0 is reserved for init_mm.
	 */
	WARN_ON(NUM_USER_ASIDS - 1 <= num_possible_cpus());
265
	atomic64_set(&asid_generation, ASID_FIRST_VERSION);
266
	asid_map = kcalloc(BITS_TO_LONGS(NUM_USER_ASIDS), sizeof(*asid_map),
267 268 269 270 271 272 273
			   GFP_KERNEL);
	if (!asid_map)
		panic("Failed to allocate bitmap for %lu ASIDs\n",
		      NUM_USER_ASIDS);

	pr_info("ASID allocator initialised with %lu entries\n", NUM_USER_ASIDS);
	return 0;
274
}
275
early_initcall(asids_init);