core.c 35.3 KB
Newer Older
1
2
3
/*
 * SPDX-License-Identifier: GPL-2.0
 *
4
 * Derived from Xenomai Cobalt, https://xenomai.org/
5
 * Copyright (C) 2001, 2008, 2018 Philippe Gerum  <rpm@xenomai.org>
6
7
8
9
10
11
12
13
14
15
16
17
 */

#include <linux/types.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/signal.h>
#include <linux/wait.h>
#include <linux/sched.h>
#include <linux/cpuidle.h>
#include <linux/mmu_context.h>
#include <asm/div64.h>
#include <asm/switch_to.h>
18
19
20
21
22
23
24
#include <evl/sched.h>
#include <evl/thread.h>
#include <evl/timer.h>
#include <evl/memory.h>
#include <evl/clock.h>
#include <evl/tick.h>
#include <evl/monitor.h>
Philippe Gerum's avatar
Philippe Gerum committed
25
#include <evl/mutex.h>
26
#include <evl/flag.h>
27
28
#include <uapi/evl/signal.h>
#include <trace/events/evl.h>
29
30
31
32
33
34
35

DEFINE_PER_CPU(struct evl_rq, evl_runqueues);
EXPORT_PER_CPU_SYMBOL_GPL(evl_runqueues);

struct cpumask evl_cpu_affinity = CPU_MASK_ALL;
EXPORT_SYMBOL_GPL(evl_cpu_affinity);

36
static struct evl_sched_class *evl_sched_topmost;
37

38
39
40
41
42
43
44
#define for_each_evl_sched_class(p)		\
	for (p = evl_sched_topmost; p; p = p->next)

static struct evl_sched_class *evl_sched_lower;

#define for_each_evl_lower_sched_class(p)	\
	for (p = evl_sched_lower; p; p = p->next)
45
46
47

static void register_one_class(struct evl_sched_class *sched_class)
{
48
49
50
51
	sched_class->next = evl_sched_topmost;
	evl_sched_topmost = sched_class;
	if (sched_class != &evl_sched_fifo)
		evl_sched_lower = sched_class;
52
53
54

	/*
	 * Classes shall be registered by increasing priority order,
55
	 * idle first and up until fifo last.
56
57
	 */
	EVL_WARN_ON(CORE, sched_class->next &&
58
		sched_class->next->weight > sched_class->weight);
59
60
61
62
63
64
}

static void register_classes(void)
{
	register_one_class(&evl_sched_idle);
	register_one_class(&evl_sched_weak);
65
#ifdef CONFIG_EVL_SCHED_QUOTA
66
	register_one_class(&evl_sched_quota);
67
68
69
#endif
#ifdef CONFIG_EVL_SCHED_TP
	register_one_class(&evl_sched_tp);
70
#endif
71
	register_one_class(&evl_sched_fifo);
72
73
74

	/* The FIFO class must be on top (see __pick_next_thread()). */
	EVL_WARN_ON_ONCE(CORE, evl_sched_topmost != &evl_sched_fifo);
75
76
}

77
#ifdef CONFIG_EVL_WATCHDOG
78

79
static unsigned long wd_timeout_arg = CONFIG_EVL_WATCHDOG_TIMEOUT;
80
81
82
83
84
85
86
module_param_named(watchdog_timeout, wd_timeout_arg, ulong, 0644);

static inline ktime_t get_watchdog_timeout(void)
{
	return ns_to_ktime(wd_timeout_arg * 1000000000ULL);
}

87
static void watchdog_handler(struct evl_timer *timer) /* oob stage stalled */
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
{
	struct evl_rq *this_rq = this_evl_rq();
	struct evl_thread *curr = this_rq->curr;

	trace_evl_watchdog_signal(curr);

	/*
	 * CAUTION: The watchdog tick might have been delayed while we
	 * were busy switching the CPU to in-band context at the
	 * trigger date eventually. Make sure that we are not about to
	 * kick the incoming root thread.
	 */
	if (curr->state & T_ROOT)
		return;

	if (curr->state & T_USER) {
104
		raw_spin_lock(&curr->lock);
105
		raw_spin_lock(&this_rq->lock);
106
		curr->info |= T_KICKED;
107
		raw_spin_unlock(&this_rq->lock);
108
		raw_spin_unlock(&curr->lock);
109
		evl_notify_thread(curr, EVL_HMDIAG_WATCHDOG, evl_nil);
110
		dovetail_send_mayday(current);
111
		printk(EVL_WARNING "watchdog triggered on CPU #%d -- runaway thread "
112
			"'%s' signaled\n", evl_rq_cpu(this_rq), curr->name);
113
114
	} else {
		printk(EVL_WARNING "watchdog triggered on CPU #%d -- runaway thread "
115
			"'%s' canceled\n", evl_rq_cpu(this_rq), curr->name);
116
117
118
119
		/*
		 * On behalf on an IRQ handler, evl_cancel_thread()
		 * would go half way cancelling the preempted
		 * thread. Therefore we manually raise T_KICKED to
120
		 * cause the next blocking call to return early in
121
122
		 * T_BREAK condition, and T_CANCELD so that @curr
		 * exits next time it invokes evl_test_cancel().
123
		 */
124
		raw_spin_lock(&curr->lock);
125
		raw_spin_lock(&this_rq->lock);
126
		curr->info |= (T_KICKED|T_CANCELD);
127
		raw_spin_unlock(&this_rq->lock);
128
		raw_spin_unlock(&curr->lock);
129
130
131
	}
}

132
#endif /* CONFIG_EVL_WATCHDOG */
133
134
135
136
137
138

static void roundrobin_handler(struct evl_timer *timer) /* hard irqs off */
{
	struct evl_rq *this_rq;

	this_rq = container_of(timer, struct evl_rq, rrbtimer);
139
	raw_spin_lock(&this_rq->lock);
140
	evl_sched_tick(this_rq);
141
	raw_spin_unlock(&this_rq->lock);
142
143
144
145
146
147
148
149
150
151
152
153
154
}

static void init_rq(struct evl_rq *rq, int cpu)
{
	struct evl_sched_class *sched_class;
	struct evl_init_thread_attr iattr;
	const char *name_fmt;

#ifdef CONFIG_SMP
	rq->cpu = cpu;
	name_fmt = "ROOT/%u";
	rq->proxy_timer_name = kasprintf(GFP_KERNEL, "[proxy-timer/%u]", cpu);
	rq->rrb_timer_name = kasprintf(GFP_KERNEL, "[rrb-timer/%u]", cpu);
Philippe Gerum's avatar
Philippe Gerum committed
155
	cpumask_clear(&rq->resched_cpus);
156
157
158
159
160
#else
	name_fmt = "ROOT";
	rq->proxy_timer_name = kstrdup("[proxy-timer]", GFP_KERNEL);
	rq->rrb_timer_name = kstrdup("[rrb-timer]", GFP_KERNEL);
#endif
161
	raw_spin_lock_init(&rq->lock);
Philippe Gerum's avatar
Philippe Gerum committed
162

163
164
165
166
167
	for_each_evl_sched_class(sched_class) {
		if (sched_class->sched_init)
			sched_class->sched_init(rq);
	}

Philippe Gerum's avatar
Philippe Gerum committed
168
169
	rq->flags = 0;
	rq->local_flags = RQ_IDLE;
170
171
172
	rq->curr = &rq->root_thread;

	/*
173
174
175
	 * No handler needed for the inband timer since proxy timer
	 * events are handled specifically by the generic timer code
	 * (do_clock_tick()).
176
	 */
177
178
	evl_init_timer_on_rq(&rq->inband_timer, &evl_mono_clock, NULL,
			rq, EVL_TIMER_IGRAVITY);
179
	evl_set_timer_name(&rq->inband_timer, rq->proxy_timer_name);
180
181
	evl_init_timer_on_rq(&rq->rrbtimer, &evl_mono_clock, roundrobin_handler,
			rq, EVL_TIMER_IGRAVITY);
182
	evl_set_timer_name(&rq->rrbtimer, rq->rrb_timer_name);
183
#ifdef CONFIG_EVL_WATCHDOG
184
185
	evl_init_timer_on_rq(&rq->wdtimer, &evl_mono_clock, watchdog_handler,
			rq, EVL_TIMER_IGRAVITY);
186
	evl_set_timer_name(&rq->wdtimer, "[watchdog]");
187
#endif /* CONFIG_EVL_WATCHDOG */
188
189
190
191
192
193
194

	evl_set_current_account(rq, &rq->root_thread.stat.account);

	/*
	 * Postpone evl_init_thread() - which sets RQ_SCHED upon
	 * setting the schedparams for the root thread - until we have
	 * enough of the runqueue initialized, so that attempting to
195
	 * reschedule from evl_exit_irq() later on is harmless.
196
197
	 */
	iattr.flags = T_ROOT;
198
	iattr.affinity = cpumask_of(cpu);
199
	iattr.observable = NULL;
200
201
202
203
204
205
206
207
208
209
	iattr.sched_class = &evl_sched_idle;
	iattr.sched_param.idle.prio = EVL_IDLE_PRIO;
	evl_init_thread(&rq->root_thread, &iattr, rq, name_fmt, cpu);

	dovetail_init_altsched(&rq->root_thread.altsched);

	list_add_tail(&rq->root_thread.next, &evl_thread_list);
	evl_nrthreads++;
}

Philippe Gerum's avatar
Philippe Gerum committed
210
static void destroy_rq(struct evl_rq *rq)
211
{
212
	evl_destroy_timer(&rq->inband_timer);
213
214
215
216
217
	evl_destroy_timer(&rq->rrbtimer);
	kfree(rq->proxy_timer_name);
	kfree(rq->rrb_timer_name);
	evl_destroy_timer(&rq->root_thread.ptimer);
	evl_destroy_timer(&rq->root_thread.rtimer);
218
#ifdef CONFIG_EVL_WATCHDOG
219
	evl_destroy_timer(&rq->wdtimer);
220
#endif /* CONFIG_EVL_WATCHDOG */
221
222
}

223
#ifdef CONFIG_EVL_DEBUG_CORE
224
225
226

void evl_disable_preempt(void)
{
227
	__evl_disable_preempt();
228
229
230
231
232
}
EXPORT_SYMBOL(evl_disable_preempt);

void evl_enable_preempt(void)
{
233
	__evl_enable_preempt();
234
235
236
}
EXPORT_SYMBOL(evl_enable_preempt);

237
#endif /* CONFIG_EVL_DEBUG_CORE */
238

Philippe Gerum's avatar
Philippe Gerum committed
239
240
241
242
243
#ifdef CONFIG_SMP

static inline
void evl_double_rq_lock(struct evl_rq *rq1, struct evl_rq *rq2)
{
244
	EVL_WARN_ON_ONCE(CORE, !hard_irqs_disabled());
Philippe Gerum's avatar
Philippe Gerum committed
245
246
247
248

	/* Prevent ABBA deadlock, always lock rqs in address order. */

	if (rq1 == rq2) {
249
		raw_spin_lock(&rq1->lock);
Philippe Gerum's avatar
Philippe Gerum committed
250
	} else if (rq1 < rq2) {
251
252
		raw_spin_lock(&rq1->lock);
		raw_spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
Philippe Gerum's avatar
Philippe Gerum committed
253
	} else {
254
255
		raw_spin_lock(&rq2->lock);
		raw_spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
Philippe Gerum's avatar
Philippe Gerum committed
256
257
258
259
260
261
	}
}

static inline
void evl_double_rq_unlock(struct evl_rq *rq1, struct evl_rq *rq2)
{
262
	raw_spin_unlock(&rq1->lock);
Philippe Gerum's avatar
Philippe Gerum committed
263
	if (rq1 != rq2)
264
		raw_spin_unlock(&rq2->lock);
Philippe Gerum's avatar
Philippe Gerum committed
265
266
267
268
269
270
271
}

static void migrate_rq(struct evl_thread *thread, struct evl_rq *dst_rq)
{
	struct evl_sched_class *sched_class = thread->sched_class;
	struct evl_rq *src_rq = thread->rq;

272
273
274
275
276
277
	/*
	 * check_cpu_affinity() might ask us to move @thread to a CPU
	 * which is not part of the oob set due to a spurious
	 * migration, this is ok. The offending thread would exit
	 * shortly after resuming.
	 */
Philippe Gerum's avatar
Philippe Gerum committed
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
	evl_double_rq_lock(src_rq, dst_rq);

	if (thread->state & T_READY) {
		evl_dequeue_thread(thread);
		thread->state &= ~T_READY;
	}

	if (sched_class->sched_migrate)
		sched_class->sched_migrate(thread, dst_rq);
	/*
	 * WARNING: the scheduling class may have just changed as a
	 * result of calling the per-class migration hook.
	 */
	thread->rq = dst_rq;

	if (!(thread->state & EVL_THREAD_BLOCK_BITS)) {
		evl_requeue_thread(thread);
		thread->state |= T_READY;
		evl_set_resched(dst_rq);
		evl_set_resched(src_rq);
	}

	evl_double_rq_unlock(src_rq, dst_rq);
}

303
/* thread->lock held, hard irqs off. @thread must be running in-band. */
Philippe Gerum's avatar
Philippe Gerum committed
304
305
void evl_migrate_thread(struct evl_thread *thread, struct evl_rq *dst_rq)
{
306
	assert_hard_lock(&thread->lock);
Philippe Gerum's avatar
Philippe Gerum committed
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324

	if (thread->rq == dst_rq)
		return;

	trace_evl_thread_migrate(thread, evl_rq_cpu(dst_rq));

	/*
	 * Timer migration is postponed until the next timeout happens
	 * for the periodic and rrb timers. The resource/periodic
	 * timer will be moved to the right CPU next time
	 * evl_prepare_timed_wait() is called for it (via
	 * evl_sleep_on()).
	 */
	migrate_rq(thread, dst_rq);

	evl_reset_account(&thread->stat.lastperiod);
}

325
static void check_cpu_affinity(struct task_struct *p) /* inband, hard irqs off */
Philippe Gerum's avatar
Philippe Gerum committed
326
327
328
329
330
{
	struct evl_thread *thread = evl_thread_from_task(p);
	int cpu = task_cpu(p);
	struct evl_rq *rq = evl_cpu_rq(cpu);

331
	raw_spin_lock(&thread->lock);
Philippe Gerum's avatar
Philippe Gerum committed
332

333
334
335
	if (likely(rq == thread->rq))
		goto out;

Philippe Gerum's avatar
Philippe Gerum committed
336
	/*
337
338
339
340
341
	 * Resync the EVL and in-band schedulers upon migration from
	 * an EVL thread, which can only happen from the in-band stage
	 * (no CPU migration from the oob stage is possible). In such
	 * an event, the CPU information kept by the EVL scheduler for
	 * that thread has become obsolete.
Philippe Gerum's avatar
Philippe Gerum committed
342
	 *
343
344
345
346
347
348
349
	 * check_cpu_affinity() detects this when the EVL thread is in
	 * flight to the oob stage. If the new CPU is not part of the
	 * oob set, mark the thread for pending cancellation but
	 * update the scheduler information nevertheless. Although
	 * some CPUs might be excluded from the oob set, all of them
	 * are capable of scheduling threads nevertheless (i.e. all
	 * runqueues are up and running).
Philippe Gerum's avatar
Philippe Gerum committed
350
351
352
353
354
355
356
	 */
	if (unlikely(!is_threading_cpu(cpu))) {
		printk(EVL_WARNING "thread %s[%d] switched to non-rt CPU%d, aborted.\n",
			thread->name, evl_get_inband_pid(thread), cpu);
		/*
		 * Can't call evl_cancel_thread() from a CPU migration
		 * point, that would break. Since we are on the wakeup
357
		 * path to oob context, just raise T_CANCELD to catch
Philippe Gerum's avatar
Philippe Gerum committed
358
359
		 * it in evl_switch_oob().
		 */
360
		raw_spin_lock(&thread->rq->lock);
Philippe Gerum's avatar
Philippe Gerum committed
361
		thread->info |= T_CANCELD;
362
		raw_spin_unlock(&thread->rq->lock);
363
364
365
366
367
368
369
370
371
	} else {
		/*
		 * If the current thread moved to a supported
		 * out-of-band CPU, which is not part of its original
		 * affinity mask, assume user wants to extend this
		 * mask.
		 */
		if (!cpumask_test_cpu(cpu, &thread->affinity))
			cpumask_set_cpu(cpu, &thread->affinity);
Philippe Gerum's avatar
Philippe Gerum committed
372
373
374
375
	}

	evl_migrate_thread(thread, rq);
out:
376
	raw_spin_unlock(&thread->lock);
Philippe Gerum's avatar
Philippe Gerum committed
377
378
379
380
381
}

#else

#define evl_double_rq_lock(__rq1, __rq2)  \
382
	EVL_WARN_ON_ONCE(CORE, !hard_irqs_disabled());
Philippe Gerum's avatar
Philippe Gerum committed
383
384
385

#define evl_double_rq_unlock(__rq1, __rq2)  do { } while (0)

386
387
static inline void check_cpu_affinity(struct task_struct *p)
{ }
Philippe Gerum's avatar
Philippe Gerum committed
388
389
390

#endif	/* CONFIG_SMP */

391
/* thread->lock + thread->rq->lock held, hard irqs off. */
392
393
void evl_putback_thread(struct evl_thread *thread)
{
394
	assert_hard_lock(&thread->lock);
395
	assert_hard_lock(&thread->rq->lock);
396

397
398
399
400
401
402
403
404
405
	if (thread->state & T_READY)
		evl_dequeue_thread(thread);
	else
		thread->state |= T_READY;

	evl_enqueue_thread(thread);
	evl_set_resched(thread->rq);
}

406
/* thread->lock + thread->rq->lock held, hard irqs off. */
407
408
409
int evl_set_thread_policy_locked(struct evl_thread *thread,
				struct evl_sched_class *sched_class,
				const union evl_sched_param *p)
410
411
412
413
414
{
	struct evl_sched_class *orig_effective_class __maybe_unused;
	bool effective;
	int ret;

415
	assert_hard_lock(&thread->lock);
416
	assert_hard_lock(&thread->rq->lock);
417

418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
	/* Check parameters early on. */
	ret = evl_check_schedparams(sched_class, thread, p);
	if (ret)
		return ret;

	/*
	 * Declaring a thread to a new scheduling class may fail, so
	 * we do that early, while the thread is still a member of the
	 * previous class. However, this also means that the
	 * declaration callback shall not do anything that might
	 * affect the previous class (such as touching thread->rq_next
	 * for instance).
	 */
	if (sched_class != thread->base_class) {
		ret = evl_declare_thread(sched_class, thread, p);
		if (ret)
			return ret;
	}

	/*
	 * As a special case, we may be called from evl_init_thread()
	 * with no previous scheduling class at all.
	 */
	if (likely(thread->base_class != NULL)) {
		if (thread->state & T_READY)
			evl_dequeue_thread(thread);

		if (sched_class != thread->base_class)
			evl_forget_thread(thread);
	}

	/*
	 * Set the base and effective scheduling parameters. However,
	 * evl_set_schedparam() will deny lowering the effective
	 * priority if a boost is undergoing, only recording the
	 * change into the base priority field in such situation.
	 */
	thread->base_class = sched_class;
	/*
	 * Referring to the effective class from a setparam() handler
	 * is wrong: make sure to break if so.
	 */
	if (EVL_DEBUG(CORE)) {
		orig_effective_class = thread->sched_class;
		thread->sched_class = NULL;
	}

	/*
	 * This is the ONLY place where calling
	 * evl_set_schedparam() is legit, sane and safe.
	 */
	effective = evl_set_schedparam(thread, p);
	if (effective) {
		thread->sched_class = sched_class;
		thread->wprio = evl_calc_weighted_prio(sched_class, thread->cprio);
	} else if (EVL_DEBUG(CORE))
		thread->sched_class = orig_effective_class;

	if (thread->state & T_READY)
		evl_enqueue_thread(thread);

	/*
	 * Make sure not to raise RQ_SCHED when setting up the root
	 * thread, so that we can't start rescheduling from
482
	 * evl_exit_irq() before all CPUs have their runqueue fully
483
484
485
486
487
488
489
490
	 * built. Filtering on T_ROOT here is correct because the root
	 * thread enters the idle class once as part of the runqueue
	 * setup process and never leaves it afterwards.
	 */
	if (!(thread->state & (T_DORMANT|T_ROOT)))
		evl_set_resched(thread->rq);
	else
		EVL_WARN_ON(CORE, (thread->state & T_ROOT) &&
491
			sched_class != &evl_sched_idle);
492
493
494
	return 0;
}

495
496
497
498
499
int evl_set_thread_policy(struct evl_thread *thread,
			struct evl_sched_class *sched_class,
			const union evl_sched_param *p)
{
	unsigned long flags;
Philippe Gerum's avatar
Philippe Gerum committed
500
	struct evl_rq *rq;
501
502
	int ret;

Philippe Gerum's avatar
Philippe Gerum committed
503
	rq = evl_get_thread_rq(thread, flags);
504
	ret = evl_set_thread_policy_locked(thread, sched_class, p);
Philippe Gerum's avatar
Philippe Gerum committed
505
	evl_put_thread_rq(thread, rq, flags);
506
507
508
509

	return ret;
}

510
/* thread->lock + thread->rq->lock held, hard irqs off. */
511
512
513
514
bool evl_set_effective_thread_priority(struct evl_thread *thread, int prio)
{
	int wprio = evl_calc_weighted_prio(thread->base_class, prio);

515
	assert_hard_lock(&thread->lock);
516
	assert_hard_lock(&thread->rq->lock);
517

518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
	thread->bprio = prio;
	if (wprio == thread->wprio)
		return true;

	/*
	 * We may not lower the effective/current priority of a
	 * boosted thread when changing the base scheduling
	 * parameters. Only evl_track_thread_policy() and
	 * evl_protect_thread_priority() may do so when dealing with PI
	 * and PP synchs resp.
	 */
	if (wprio < thread->wprio && (thread->state & T_BOOST))
		return false;

	thread->cprio = prio;

	trace_evl_thread_set_current_prio(thread);

	return true;
}

539
/* thread->lock + target->lock held, hard irqs off */
540
void evl_track_thread_policy(struct evl_thread *thread,
541
			struct evl_thread *target)
542
543
544
{
	union evl_sched_param param;

545
546
	assert_hard_lock(&thread->lock);
	assert_hard_lock(&target->lock);
547

Philippe Gerum's avatar
Philippe Gerum committed
548
549
	evl_double_rq_lock(thread->rq, target->rq);

550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
	/*
	 * Inherit (or reset) the effective scheduling class and
	 * priority of a thread. Unlike evl_set_thread_policy(), this
	 * routine is allowed to lower the weighted priority with no
	 * restriction, even if a boost is undergoing.
	 */
	if (thread->state & T_READY)
		evl_dequeue_thread(thread);
	/*
	 * Self-targeting means to reset the scheduling policy and
	 * parameters to the base settings. Otherwise, make thread
	 * inherit the scheduling parameters from target.
	 */
	if (target == thread) {
		thread->sched_class = thread->base_class;
		evl_track_priority(thread, NULL);
		/*
		 * Per SuSv2, resetting the base scheduling parameters
		 * should not move the thread to the tail of its
Philippe Gerum's avatar
Philippe Gerum committed
569
		 * priority group, which makes sense.
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
		 */
		if (thread->state & T_READY)
			evl_requeue_thread(thread);

	} else {
		evl_get_schedparam(target, &param);
		thread->sched_class = target->sched_class;
		evl_track_priority(thread, &param);
		if (thread->state & T_READY)
			evl_enqueue_thread(thread);
	}

	trace_evl_thread_set_current_prio(thread);

	evl_set_resched(thread->rq);
585

Philippe Gerum's avatar
Philippe Gerum committed
586
	evl_double_rq_unlock(thread->rq, target->rq);
587
588
}

589
/* thread->lock, hard irqs off */
590
591
void evl_protect_thread_priority(struct evl_thread *thread, int prio)
{
592
	assert_hard_lock(&thread->lock);
593

594
	raw_spin_lock(&thread->rq->lock);
595

596
597
	/*
	 * Apply a PP boost by changing the effective priority of a
598
	 * thread, forcing it to the FIFO class. Like
599
600
601
602
603
604
605
606
607
608
609
	 * evl_track_thread_policy(), this routine is allowed to lower
	 * the weighted priority with no restriction, even if a boost
	 * is undergoing.
	 *
	 * This routine only deals with active boosts, resetting the
	 * base priority when leaving a PP boost is obtained by a call
	 * to evl_track_thread_policy().
	 */
	if (thread->state & T_READY)
		evl_dequeue_thread(thread);

610
	thread->sched_class = &evl_sched_fifo;
611
612
613
614
615
616
617
618
	evl_ceil_priority(thread, prio);

	if (thread->state & T_READY)
		evl_enqueue_thread(thread);

	trace_evl_thread_set_current_prio(thread);

	evl_set_resched(thread->rq);
619

620
	raw_spin_unlock(&thread->rq->lock);
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
}

void evl_init_schedq(struct evl_multilevel_queue *q)
{
	int prio;

	q->elems = 0;
	bitmap_zero(q->prio_map, EVL_MLQ_LEVELS);

	for (prio = 0; prio < EVL_MLQ_LEVELS; prio++)
		INIT_LIST_HEAD(q->heads + prio);
}

struct evl_thread *evl_get_schedq(struct evl_multilevel_queue *q)
{
	struct evl_thread *thread;
	struct list_head *head;
	int idx;

640
	if (evl_schedq_is_empty(q))
641
642
643
644
645
		return NULL;

	idx = evl_get_schedq_weight(q);
	head = q->heads + idx;
	thread = list_first_entry(head, struct evl_thread, rq_next);
646
	__evl_del_schedq(q, &thread->rq_next, idx);
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664

	return thread;
}

struct evl_thread *
evl_lookup_schedq(struct evl_multilevel_queue *q, int prio)
{
	struct list_head *head;
	int idx;

	idx = get_qindex(q, prio);
	head = q->heads + idx;
	if (list_empty(head))
		return NULL;

	return list_first_entry(head, struct evl_thread, rq_next);
}

665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
static inline void enter_inband(struct evl_thread *root)
{
#ifdef CONFIG_EVL_WATCHDOG
	evl_stop_timer(&evl_thread_rq(root)->wdtimer);
#endif
}

static inline void leave_inband(struct evl_thread *root)
{
#ifdef CONFIG_EVL_WATCHDOG
	evl_start_timer(&evl_thread_rq(root)->wdtimer,
			evl_abs_timeout(&evl_thread_rq(root)->wdtimer,
					get_watchdog_timeout()),
			EVL_INFINITE);
#endif
}

/* oob stalled. */
static irqreturn_t oob_reschedule_interrupt(int irq, void *dev_id)
{
	trace_evl_reschedule_ipi(this_evl_rq());

	/* Will reschedule from evl_exit_irq(). */

	return IRQ_HANDLED;
}

static struct evl_thread *lookup_fifo_class(struct evl_rq *rq)
693
{
694
	struct evl_multilevel_queue *q = &rq->fifo.runnable;
695
696
697
698
	struct evl_thread *thread;
	struct list_head *head;
	int idx;

699
	if (evl_schedq_is_empty(q))
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
		return NULL;

	/*
	 * Some scheduling policies may be implemented as variants of
	 * the core SCHED_FIFO class, sharing its runqueue
	 * (e.g. SCHED_QUOTA). This means that we have to do some
	 * cascading to call the right pick handler eventually.
	 */
	idx = evl_get_schedq_weight(q);
	head = q->heads + idx;

	/*
	 * The active class (i.e. ->sched_class) is the one currently
	 * queuing the thread, reflecting any priority boost due to
	 * PI.
	 */
	thread = list_first_entry(head, struct evl_thread, rq_next);
717
	if (unlikely(thread->sched_class != &evl_sched_fifo))
718
719
		return thread->sched_class->sched_pick(rq);

720
	__evl_del_schedq(q, &thread->rq_next, idx);
721
722
723
724

	return thread;
}

Philippe Gerum's avatar
Philippe Gerum committed
725
726
static inline void set_next_running(struct evl_rq *rq,
				struct evl_thread *next)
727
{
Philippe Gerum's avatar
Philippe Gerum committed
728
729
	next->state &= ~T_READY;
	if (next->state & T_RRB)
730
		evl_start_timer(&rq->rrbtimer,
Philippe Gerum's avatar
Philippe Gerum committed
731
				evl_abs_timeout(&rq->rrbtimer, next->rrperiod),
732
733
734
735
736
				EVL_INFINITE);
	else
		evl_stop_timer(&rq->rrbtimer);
}

737
static struct evl_thread *__pick_next_thread(struct evl_rq *rq)
738
{
739
	struct evl_sched_class *sched_class;
740
	struct evl_thread *curr = rq->curr;
Philippe Gerum's avatar
Philippe Gerum committed
741
	struct evl_thread *next;
742
743
744
745
746
747
748

	/*
	 * We have to switch the current thread out if a blocking
	 * condition is raised for it. Otherwise, check whether
	 * preemption is allowed.
	 */
	if (!(curr->state & (EVL_THREAD_BLOCK_BITS | T_ZOMBIE))) {
749
		if (evl_preempt_count() > 0) {
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
			evl_set_self_resched(rq);
			return curr;
		}
		/*
		 * Push the current thread back to the run queue of
		 * the scheduling class it belongs to, if not yet
		 * linked to it (T_READY tells us if it is).
		 */
		if (!(curr->state & T_READY)) {
			evl_requeue_thread(curr);
			curr->state |= T_READY;
		}
	}

	/*
765
	 * Find the next runnable thread with the highest priority
Philippe Gerum's avatar
Philippe Gerum committed
766
	 * amongst all scheduling classes, scanned by decreasing
767
768
769
	 * weight. We start from the FIFO class which is always on top
	 * without going through any indirection; if no thread is
	 * runnable there, poll the lower classes.
770
	 */
771
772
773
774
775
	next = lookup_fifo_class(rq);
	if (likely(next))
		return next;

	for_each_evl_lower_sched_class(sched_class) {
Philippe Gerum's avatar
Philippe Gerum committed
776
		next = sched_class->sched_pick(rq);
777
		if (next)
Philippe Gerum's avatar
Philippe Gerum committed
778
			return next;
779
780
	}

Philippe Gerum's avatar
Philippe Gerum committed
781
782
783
	return NULL; /* NOT REACHED (idle class). */
}

784
/* rq->curr->lock + rq->lock held, hard irqs off. */
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
static struct evl_thread *pick_next_thread(struct evl_rq *rq)
{
	struct oob_mm_state *oob_mm;
	struct evl_thread *next;

	for (;;) {
		next = __pick_next_thread(rq);
		oob_mm = next->oob_mm;
		if (unlikely(!oob_mm)) /* Includes the root thread. */
			break;
		/*
		 * Obey any pending request for a ptsync freeze.
		 * Either we freeze @next before a sigwake event lifts
		 * T_PTSYNC, setting T_PTSTOP, or after in which case
		 * we already have T_PTSTOP set so we don't have to
		 * raise T_PTSYNC. The basic assumption is that we
		 * should get SIGSTOP/SIGTRAP for any thread involved.
		 */
		if (likely(!test_bit(EVL_MM_PTSYNC_BIT, &oob_mm->flags)))
			break;	/* Fast and most likely path. */
		if (next->info & (T_PTSTOP|T_PTSIG|T_KICKED))
			break;
		/*
		 * NOTE: We hold next->rq->lock by construction, so
		 * changing next->state is ok despite that we don't
		 * hold next->lock. This properly serializes with
		 * evl_kick_thread() which might raise T_PTSTOP.
		 */
		next->state |= T_PTSYNC;
		next->state &= ~T_READY;
	}

	set_next_running(rq, next);

	return next;
}

Philippe Gerum's avatar
Philippe Gerum committed
822
823
824
825
static inline void prepare_rq_switch(struct evl_rq *this_rq,
				struct evl_thread *next)
{
	if (irq_pipeline_debug_locking())
826
		spin_release(&this_rq->lock.rlock.dep_map,
Philippe Gerum's avatar
Philippe Gerum committed
827
828
			_THIS_IP_);
#ifdef CONFIG_DEBUG_SPINLOCK
829
	this_rq->lock.rlock.owner = next->altsched.task;
Philippe Gerum's avatar
Philippe Gerum committed
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
#endif
}

static inline void finish_rq_switch(bool inband_tail, unsigned long flags)
{
	struct evl_rq *this_rq = this_evl_rq();

	EVL_WARN_ON(CORE, this_rq->curr->state & EVL_THREAD_BLOCK_BITS);

	/*
	 * Check whether we are completing a transition to the inband
	 * stage for the current task, i.e.:
	 *
	 * irq_work_queue() ->
	 *        IRQ:wake_up_process() ->
	 *                         schedule() ->
	 *                               back from dovetail_context_switch()
	 */
	if (likely(!inband_tail)) {
		if (irq_pipeline_debug_locking())
850
			spin_acquire(&this_rq->lock.rlock.dep_map,
Philippe Gerum's avatar
Philippe Gerum committed
851
				0, 0, _THIS_IP_);
852
		raw_spin_unlock_irqrestore(&this_rq->lock, flags);
Philippe Gerum's avatar
Philippe Gerum committed
853
854
855
856
857
858
859
	}
}

static inline void finish_rq_switch_from_inband(void)
{
	struct evl_rq *this_rq = this_evl_rq();

860
	assert_hard_lock(&this_rq->lock);
Philippe Gerum's avatar
Philippe Gerum committed
861
862

	if (irq_pipeline_debug_locking())
863
		spin_acquire(&this_rq->lock.rlock.dep_map,
Philippe Gerum's avatar
Philippe Gerum committed
864
865
			0, 0, _THIS_IP_);

866
	raw_spin_unlock_irq(&this_rq->lock);
Philippe Gerum's avatar
Philippe Gerum committed
867
868
}

869
/* hard irqs off. */
Philippe Gerum's avatar
Philippe Gerum committed
870
871
872
873
874
875
876
static inline bool test_resched(struct evl_rq *this_rq)
{
	bool need_resched = evl_need_resched(this_rq);

#ifdef CONFIG_SMP
	/* Send resched IPI to remote CPU(s). */
	if (unlikely(!cpumask_empty(&this_rq->resched_cpus))) {
877
		irq_send_oob_ipi(RESCHEDULE_OOB_IPI, &this_rq->resched_cpus);
Philippe Gerum's avatar
Philippe Gerum committed
878
879
880
881
882
883
884
885
		cpumask_clear(&this_rq->resched_cpus);
		this_rq->local_flags &= ~RQ_SCHED;
	}
#endif
	if (need_resched)
		this_rq->flags &= ~RQ_SCHED;

	return need_resched;
886
887
}

888
/*
889
890
891
892
893
 * CAUTION: curr->altsched.task may be unsynced and even stale if
 * (this_rq->curr == &this_rq->root_thread), since the task logged by
 * dovetail_context_switch() may not still be the current one. Always
 * use "current" for disambiguating if you intend to refer to the
 * running inband task.
894
 */
895
void __evl_schedule(void) /* oob or/and hard irqs off (CPU migration-safe) */
896
{
897
	struct evl_rq *this_rq = this_evl_rq();
898
	struct evl_thread *prev, *next, *curr;
899
	bool leaving_inband, inband_tail;
900
901
	unsigned long flags;

902
	if (EVL_WARN_ON_ONCE(CORE, running_inband() && !hard_irqs_disabled()))
903
904
		return;

905
906
	trace_evl_schedule(this_rq);

907
	flags = hard_local_irq_save();
908
909

	/*
910
911
912
913
914
	 * Check whether we have a pending priority ceiling request to
	 * commit before putting the current thread to sleep.
	 * evl_current() may differ from rq->curr only if rq->curr ==
	 * &rq->root_thread. Testing T_USER eliminates this case since
	 * a root thread never bears this bit.
915
	 */
916
917
918
919
	curr = this_rq->curr;
	if (curr->state & T_USER)
		evl_commit_monitor_ceiling();

Philippe Gerum's avatar
Philippe Gerum committed
920
921
922
923
924
	/*
	 * Only holding this_rq->lock is required for test_resched(),
	 * but we grab curr->lock in advance in order to keep the
	 * locking order safe from ABBA deadlocking.
	 */
925
	raw_spin_lock(&curr->lock);
926
	raw_spin_lock(&this_rq->lock);
927

928
	if (unlikely(!test_resched(this_rq))) {
929
		raw_spin_unlock(&this_rq->lock);
930
		raw_spin_unlock_irqrestore(&curr->lock, flags);
931
932
		return;
	}
933

934
	next = pick_next_thread(this_rq);
935
936
	if (next == curr) {
		if (unlikely(next->state & T_ROOT)) {
Philippe Gerum's avatar
Philippe Gerum committed
937
			if (this_rq->local_flags & RQ_TPROXY)
938
				evl_notify_proxy_tick(this_rq);
Philippe Gerum's avatar
Philippe Gerum committed
939
			if (this_rq->local_flags & RQ_TDEFER)
940
941
				evl_program_local_tick(&evl_mono_clock);
		}
942
		raw_spin_unlock(&this_rq->lock);
943
		raw_spin_unlock_irqrestore(&curr->lock, flags);
944
		return;
945
946
947
948
949
	}

	prev = curr;
	trace_evl_switch_context(prev, next);
	this_rq->curr = next;
950
	leaving_inband = false;
951
952

	if (prev->state & T_ROOT) {
953
954
		leave_inband(prev);
		leaving_inband = true;
955
	} else if (next->state & T_ROOT) {
Philippe Gerum's avatar
Philippe Gerum committed
956
		if (this_rq->local_flags & RQ_TPROXY)
957
			evl_notify_proxy_tick(this_rq);
Philippe Gerum's avatar
Philippe Gerum committed
958
		if (this_rq->local_flags & RQ_TDEFER)
959
			evl_program_local_tick(&evl_mono_clock);
960
		enter_inband(next);
961
962
963
964
	}

	evl_switch_account(this_rq, &next->stat.account);
	evl_inc_counter(&next->stat.csw);
965
	raw_spin_unlock(&prev->lock);
Philippe Gerum's avatar
Philippe Gerum committed
966
967

	prepare_rq_switch(this_rq, next);
968
969
	inband_tail = dovetail_context_switch(&prev->altsched,
					&next->altsched, leaving_inband);
Philippe Gerum's avatar
Philippe Gerum committed
970
971
972
973
	finish_rq_switch(inband_tail, flags);
}
EXPORT_SYMBOL_GPL(__evl_schedule);

974
/* this_rq->lock held, hard irqs off. */
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
static void start_ptsync_locked(struct evl_thread *stopper,
				struct evl_rq *this_rq)
{
	struct oob_mm_state *oob_mm = stopper->oob_mm;

	if (!test_and_set_bit(EVL_MM_PTSYNC_BIT, &oob_mm->flags)) {
#ifdef CONFIG_SMP
		cpumask_copy(&this_rq->resched_cpus, &evl_oob_cpus);
		cpumask_clear_cpu(raw_smp_processor_id(), &this_rq->resched_cpus);
#endif
		evl_set_self_resched(this_rq);
	}
}

void evl_start_ptsync(struct evl_thread *stopper)
{
	struct evl_rq *this_rq;
	unsigned long flags;

	if (EVL_WARN_ON(CORE, !(stopper->state & T_USER)))
		return;

997
	flags = hard_local_irq_save();
998
	this_rq = this_evl_rq();
999
	raw_spin_lock(&this_rq->lock);
1000
	start_ptsync_locked(stopper, this_rq);
1001
	raw_spin_unlock_irqrestore(&this_rq->lock, flags);
1002
1003
}

Philippe Gerum's avatar
Philippe Gerum committed
1004
1005
1006
1007
void resume_oob_task(struct task_struct *p) /* inband, oob stage stalled */
{
	struct evl_thread *thread = evl_thread_from_task(p);

1008
	/*
1009
1010
1011
	 * Dovetail calls us with hard irqs off, oob stage
	 * stalled. Clear the stall bit which we don't use for
	 * protection but keep hard irqs off.
1012
	 */
1013
	unstall_oob();
1014
1015
	check_cpu_affinity(p);
	evl_release_thread(thread, T_INBAND, 0);
1016
1017
1018
1019
	/*
	 * If T_PTSTOP is set, pick_next_thread() is not allowed to
	 * freeze @thread while in flight to the out-of-band stage.
	 */
Philippe Gerum's avatar
Philippe Gerum committed
1020
	evl_schedule();
1021
	stall_oob();
Philippe Gerum's avatar
Philippe Gerum committed
1022
1023
1024
1025
}

int evl_switch_oob(void)
{
1026
	struct evl_thread *curr = evl_current();
Philippe Gerum's avatar
Philippe Gerum committed
1027
	struct task_struct *p = current;
1028
	unsigned long flags;
Philippe Gerum's avatar
Philippe Gerum committed
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
	int ret;

	inband_context_only();

	if (curr == NULL)
		return -EPERM;

	if (signal_pending(p))
		return -ERESTARTSYS;

1039
	trace_evl_switch_oob(curr);
Philippe Gerum's avatar
Philippe Gerum committed
1040
1041
1042
1043
1044
1045
1046
1047
1048

	evl_clear_sync_uwindow(curr, T_INBAND);

	ret = dovetail_leave_inband();
	if (ret) {
		evl_test_cancel();
		evl_set_sync_uwindow(curr, T_INBAND);
		return ret;
	}
1049

1050
1051
1052
1053
1054
1055
1056
	/*
	 * On success, dovetail_leave_inband() stalls the oob stage
	 * before returning to us: clear this stall bit since we don't
	 * use it for protection but keep hard irqs off.
	 */
	unstall_oob();

1057
	/*
Philippe Gerum's avatar
Philippe Gerum committed
1058
1059
1060
	 * The current task is now running on the out-of-band
	 * execution stage, scheduled in by the latest call to
	 * __evl_schedule() on this CPU: we must be holding the
1061
	 * runqueue lock and hard irqs must be off.
Philippe Gerum's avatar
Philippe Gerum committed
1062
1063
	 */
	oob_context_only();
1064

1065
	finish_rq_switch_from_inband();
Philippe Gerum's avatar
Philippe Gerum committed
1066
1067
1068

	trace_evl_switched_oob(curr);

1069
1070
1071
1072
1073
1074
1075
1076
	/*
	 * In case check_cpu_affinity() caught us resuming oob from a
	 * wrong CPU (i.e. outside of the oob set), we have T_CANCELD
	 * set. Check and bail out if so.
	 */
	if (curr->info & T_CANCELD)
		evl_test_cancel();

Philippe Gerum's avatar
Philippe Gerum committed
1077
	/*
1078
1079
1080
1081
1082
1083
	 * Since handle_sigwake_event()->evl_kick_thread() won't set
	 * T_KICKED unless T_INBAND is cleared, a signal received
	 * during the stage transition process might have gone
	 * unnoticed. Recheck for signals here and raise T_KICKED if
	 * some are pending, so that we switch back in-band asap for
	 * handling them.
Philippe Gerum's avatar
Philippe Gerum committed
1084
1085
	 */
	if (signal_pending(p)) {
1086
		raw_spin_lock_irqsave(&curr->rq->lock, flags);
1087
		curr->info |= T_KICKED;
1088
		raw_spin_unlock_irqrestore(&curr->rq->lock, flags);
Philippe Gerum's avatar
Philippe Gerum committed
1089
1090
1091
1092
1093
1094
1095
1096
1097
	}

	return 0;
}
EXPORT_SYMBOL_GPL(evl_switch_oob);

void evl_switch_inband(int cause)
{
	struct evl_thread *curr = evl_current();
1098
1099
	struct evl_rq *this_rq;
	bool notify;
Philippe Gerum's avatar
Philippe Gerum committed
1100
1101
1102

	oob_context_only();

1103
	trace_evl_switch_inband(cause);
Philippe Gerum's avatar
Philippe Gerum committed
1104
1105
1106
1107
1108

	/*
	 * This is the only location where we may assert T_INBAND for
	 * a thread. Basic assumption: switching to the inband stage
	 * only applies to the current thread running out-of-band on
1109
	 * this CPU. See caveat about dovetail_leave_oob() below.
1110
	 */
1111
	hard_local_irq_disable();
Philippe Gerum's avatar
Philippe Gerum committed
1112
	irq_work_queue(&curr->inband_work);
1113

1114
	raw_spin_lock(&curr->lock);
1115
	this_rq = curr->rq;
1116
	raw_spin_lock(&this_rq->lock);
1117

Philippe Gerum's avatar
Philippe Gerum committed
1118
1119
1120
1121
	if (curr->state & T_READY) {
		evl_dequeue_thread(curr);
		curr->state &= ~T_READY;
	}
1122

Philippe Gerum's avatar
Philippe Gerum committed
1123
1124
	curr->state |= T_INBAND;
	curr->local_info &= ~T_SYSRST;
1125
	notify = curr->state & T_USER && cause > EVL_HMDIAG_NONE;
1126
1127
1128

	/*
	 * If we are initiating the ptsync sequence on breakpoint or
1129
1130
	 * SIGSTOP/SIGINT is pending, do not send any HM notification
	 * since switching in-band is ok.
1131
	 */
1132
	if (cause == EVL_HMDIAG_TRAP) {
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
		curr->info |= T_PTSTOP;
		curr->info &= ~T_PTJOIN;
		start_ptsync_locked(curr, this_rq);
	} else if (curr->info & T_PTSIG) {
		curr->info &= ~T_PTSIG;
		notify = false;
	}

	curr->info &= ~EVL_THREAD_INFO_MASK;

	evl_set_resched(this_rq);

1145
	raw_spin_unlock(&this_rq->lock);
1146
	raw_spin_unlock(&curr->lock);
1147
1148
1149
1150
1151
1152
1153
1154

	/*
	 * CAVEAT: dovetail_leave_oob() must run _before_ the in-band
	 * kernel is allowed to take interrupts again, so that
	 * try_to_wake_up() does not block the wake up request for the
	 * switching thread as a result of testing
	 * task_is_off_stage().
	 */
1155
	dovetail_leave_oob();
1156

Philippe Gerum's avatar
Philippe Gerum committed
1157
1158
1159
1160
1161
	__evl_schedule();
	/*
	 * this_rq()->lock was released when the root thread resumed
	 * from __evl_schedule() (i.e. inband_tail path).
	 */
1162
	hard_local_irq_enable();
Philippe Gerum's avatar
Philippe Gerum committed
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
	dovetail_resume_inband();

	/*
	 * Basic sanity check after an expected transition to in-band
	 * context.
	 */
	EVL_WARN(CORE, !running_inband(),
		"evl_switch_inband() failed for thread %s[%d]",
		curr->name, evl_get_inband_pid(curr));

	/* Account for switch to in-band context. */
	evl_inc_counter(&curr->stat.isw);

	trace_evl_switched_inband(curr);

	/*
	 * When switching to in-band context, we check for propagating
	 * the current EVL schedparams that might have been set for
	 * current while running in OOB context.
	 *
	 * CAUTION: This obviously won't update the schedparams cached
	 * by the glibc for the caller in user-space, but this is the
	 * deal: we don't switch threads which issue
	 * EVL_THRIOC_SET_SCHEDPARAM to in-band mode, but then only
	 * the kernel side will be aware of the change, and glibc
	 * might cache obsolete information.
	 */
	evl_propagate_schedparam_change(curr);
1191

1192
	if (notify) {
Philippe Gerum's avatar
Philippe Gerum committed
1193
1194
		/*
		 * Help debugging spurious stage switches by sending
1195
		 * an HM event.
Philippe Gerum's avatar
Philippe Gerum committed
1196
		 */
1197
1198
1199
		if (curr->state & T_WOSS)
			evl_notify_thread(curr, cause, evl_nil);

Philippe Gerum's avatar
Philippe Gerum committed
1200
1201
		/* May check for locking inconsistency too. */
		if (curr->state & T_WOLI)
1202
			evl_detect_boost_drop();
Philippe Gerum's avatar
Philippe Gerum committed
1203
1204
1205
1206
	}

	/* @curr is now running inband. */
	evl_sync_uwindow(curr);
1207
}
Philippe Gerum's avatar
Philippe Gerum committed
1208
EXPORT_SYMBOL_GPL(evl_switch_inband);
1209
1210
1211

struct evl_sched_class *
evl_find_sched_class(union evl_sched_param *param,
1212
1213
		const struct evl_sched_attrs *attrs,
		ktime_t *tslice_r)
1214
1215
1216
1217
1218
1219
1220
1221
{
	struct evl_sched_class *sched_class;
	int prio, policy;
	ktime_t tslice;

	policy = attrs->sched_policy;
	prio = attrs->sched_priority;
	tslice = EVL_INFINITE;
1222
1223
	sched_class = &evl_sched_fifo;
	param->fifo.prio = prio;
1224
1225
1226
1227

	switch (policy) {
	case SCHED_NORMAL:
		if (prio)
1228
			return ERR_PTR(-EINVAL);
1229
		fallthrough;