1
#include "internal.h"
2
3
void wake_sleepers(u64 now) {
4
for (;;) {
5
sched_thread_t *thread = sleep_heap_top();
6
if (!thread) {
7
break;
8
}9
10
if (!thread->sleep_queued || !thread->wake_tick) {
11
sleep_heap_remove_at(0);
12
continue;
13
}14
15
if (thread->wake_tick > now) {
16
break;
17
}18
19
sleep_heap_remove(thread);
20
bool had_deadline = thread->wait_deadline_tick != 0;
21
thread->wake_tick = 0;
22
thread->wait_deadline_tick = 0;
23
24
thread_state_t state = thread_get_state(thread);
25
if (state == THREAD_SLEEPING) {
26
if (thread->in_wait_queue) {
27
wq_dequeue(thread);
28
}29
30
thread->wait_result = (u8)SCHED_WAIT_TIMEOUT;
31
__atomic_fetch_add(&sched_state.metrics.wait_timeout_count, 1, __ATOMIC_RELAXED);
32
33
if (sched_reclaim_handoff(thread)) {
34
thread_set_state(thread, THREAD_READY);
35
enqueue_ipi(thread, true);
36
continue;
37
}38
39
if (thread_on_local_cpu(thread)) {
40
sched_nudge_thread(thread);
41
continue;
42
}43
44
thread_unclaim(thread);
45
thread_set_state(thread, THREAD_READY);
46
47
enqueue_ipi(thread, true);
48
49
continue;
50
}51
52
if (state == THREAD_RUNNING) {
53
sched_repair_thread(thread, true);
54
continue;
55
}56
57
(void)had_deadline;
58
}59
}60
61
void sched_wake_sleepers(u64 now) {
62
unsigned long sched_flags = 0;
63
if (!sched_lock_try_save(&sched_flags)) {
64
return;
65
}66
67
wake_sleepers(now);
68
sched_lock_restore(sched_flags);
69
}70
71
static bool wq_unlink(sched_wait_queue_t *queue, sched_thread_t *thread) {
72
if (!queue || !queue->list || !thread) {
73
return false;
74
}75
76
list_node_t *target = &thread->wait_node;
77
78
if (target->owner == queue->list) {
79
list_node_t *prev = target->prev;
80
list_node_t *next = target->next;
81
82
if (prev) {
83
prev->next = next;
84
} else {
85
queue->list->head = next;
86
}87
88
if (next) {
89
next->prev = prev;
90
} else {
91
queue->list->tail = prev;
92
}93
94
if (queue->list->length) {
95
queue->list->length--;
96
}97
if (queue->waiter_count) {
98
queue->waiter_count--;
99
}100
101
target->next = NULL;
102
target->prev = NULL;
103
target->owner = NULL;
104
105
return true;
106
}107
108
for (list_node_t *it = queue->list->head; it; it = it->next) {
109
if (it != target && it->data != thread) {
110
continue;
111
}112
113
if (it->prev) {
114
it->prev->next = it->next;
115
} else {
116
queue->list->head = it->next;
117
}118
119
if (it->next) {
120
it->next->prev = it->prev;
121
} else {
122
queue->list->tail = it->prev;
123
}124
125
if (queue->list->length) {
126
queue->list->length--;
127
}128
if (queue->waiter_count) {
129
queue->waiter_count--;
130
}131
132
it->next = NULL;
133
it->prev = NULL;
134
it->owner = NULL;
135
136
return true;
137
}138
139
return false;
140
}141
142
void wq_dequeue(sched_thread_t *thread) {
143
if (!thread || !thread->in_wait_queue || !thread->blocked_on) {
144
return;
145
}146
147
if (!thread->blocked_on->list) {
148
thread->in_wait_queue = false;
149
thread->blocked_on = NULL;
150
return;
151
}152
153
sched_wait_queue_t *queue = thread->blocked_on;
154
wq_unlink(queue, thread);
155
156
thread->wait_node.next = NULL;
157
thread->wait_node.prev = NULL;
158
thread->wait_node.owner = NULL;
159
thread->in_wait_queue = false;
160
thread->blocked_on = NULL;
161
}162
163
void wq_remove(sched_thread_t *thread) {
164
if (!thread || !thread->in_wait_queue || !thread->blocked_on) {
165
return;
166
}167
168
unsigned long flags = sched_lock_save();
169
wq_dequeue(thread);
170
sched_lock_restore(flags);
171
}172
173
bool wait_running(sched_thread_t *self) {
174
if (!self) {
175
return false;
176
}177
178
bool resched_kicked = false;
179
180
for (;;) {
181
thread_state_t state = thread_get_state(self);
182
if (state == THREAD_RUNNING) {
183
if (sched_local_current() == self) {
184
return true;
185
}186
}187
188
if (!sched_is_running()) {
189
return false;
190
}191
192
if (state == THREAD_ZOMBIE) {
193
return false;
194
}195
196
if (!resched_kicked) {
197
force_resched();
198
resched_kicked = true;
199
}200
201
sched_spin_wait();
202
}203
}204
205
void sched_discard_thread(sched_thread_t *thread) {
206
if (!thread || sched_thread_is_idle(thread)) {
207
return;
208
}209
210
rq_remove(thread);
211
wq_remove(thread);
212
sleep_heap_remove(thread);
213
thread_unclaim(thread);
214
215
if (thread->in_zombie_list && sched_state.procs.zombie_list) {
216
unsigned long flags = sched_lock_save();
217
218
if (thread->in_zombie_list) {
219
list_remove(sched_state.procs.zombie_list, &thread->zombie_node);
220
thread->in_zombie_list = false;
221
}222
223
sched_lock_restore(flags);
224
}225
226
thread_cleanup(thread);
227
thread_put(thread);
228
}229
230
void sched_make_runnable(sched_thread_t *thread) {
231
if (!thread || sched_thread_is_idle(thread)) {
232
return;
233
}234
235
unsigned long flags = sched_lock_save();
236
237
wq_dequeue(thread);
238
sleep_heap_remove(thread);
239
thread->wake_tick = 0;
240
thread->wait_deadline_tick = 0;
241
thread->wait_result = (u8)SCHED_WAIT_WOKEN;
242
243
if (sched_reclaim_handoff(thread)) {
244
thread_set_state(thread, THREAD_READY);
245
enqueue_thread(thread);
246
sched_lock_restore(flags);
247
return;
248
}249
250
if (thread_on_local_cpu(thread)) {
251
sched_nudge_thread(thread);
252
sched_lock_restore(flags);
253
return;
254
}255
256
if (thread_get_state(thread) == THREAD_RUNNING) {
257
sched_repair_thread(thread, true);
258
sched_lock_restore(flags);
259
return;
260
}261
262
thread_unclaim(thread);
263
thread_set_state(thread, THREAD_READY);
264
enqueue_thread(thread);
265
266
sched_lock_restore(flags);
267
}268
269
void sched_unblock_thread(sched_thread_t *thread) {
270
if (!thread || sched_thread_is_idle(thread)) {
271
return;
272
}273
274
unsigned long flags = sched_lock_save();
275
276
wq_dequeue(thread);
277
sleep_heap_remove(thread);
278
279
thread_state_t state = thread_get_state(thread);
280
if (state == THREAD_SLEEPING || state == THREAD_READY) {
281
thread->wake_tick = 0;
282
thread->wait_deadline_tick = 0;
283
284
thread->wait_result = (u8)(((thread->wait_flags & SCHED_WAIT_INTERRUPTIBLE) && sched_signal_has_pending(thread))
285
? SCHED_WAIT_INTR286
: SCHED_WAIT_WOKEN);
287
288
if (sched_reclaim_handoff(thread)) {
289
thread_set_state(thread, THREAD_READY);
290
enqueue_thread(thread);
291
sched_lock_restore(flags);
292
return;
293
}294
295
if (thread_on_local_cpu(thread)) {
296
sched_nudge_thread(thread);
297
sched_lock_restore(flags);
298
return;
299
}300
301
thread_unclaim(thread);
302
thread_set_state(thread, THREAD_READY);
303
enqueue_thread(thread);
304
} else if (state == THREAD_RUNNING) {
305
sched_repair_thread(thread, true);
306
}307
308
sched_lock_restore(flags);
309
}310
311
void sched_stop_thread(sched_thread_t *thread, int signum) {
312
if (!thread || sched_thread_is_idle(thread)) {
313
return;
314
}315
316
unsigned long flags = sched_lock_save();
317
thread_state_t state = thread_get_state(thread);
318
319
if (state == THREAD_ZOMBIE || state == THREAD_STOPPED) {
320
sched_lock_restore(flags);
321
return;
322
}323
324
bool request_local_resched = false;
325
size_t request_remote_resched_cpu = MAX_CORES;
326
bool stopping_current = (thread == sched_local_current());
327
bool owned_running_thread =
328
(!stopping_current && state == THREAD_RUNNING && thread_cpu(thread) >= 0 && thread_is_owned(thread));
329
330
if (owned_running_thread) {
331
if (signum > 0 && signum < NSIG) {
332
u32 mask = 1u << (signum - 1);
333
__atomic_fetch_or(&thread->signal_pending, mask, __ATOMIC_ACQ_REL);
334
}335
336
thread->stop_signal = signum;
337
thread->stop_reported = false;
338
339
size_t target_cpu = (size_t)thread_cpu(thread);
340
if (target_cpu == sched_cpu_id()) {
341
request_local_resched = true;
342
} else if (target_cpu < MAX_CORES) {
343
request_remote_resched_cpu = target_cpu;
344
}345
346
sched_lock_restore(flags);
347
if (request_local_resched) {
348
sched_request_resched_local();
349
} else {
350
bool woke_remote = (request_remote_resched_cpu < MAX_CORES && wake_cpu(request_remote_resched_cpu));
351
352
if (woke_remote) {
353
__atomic_fetch_add(&sched_state.metrics.wake_ipi, 1, __ATOMIC_RELAXED);
354
}355
}356
return;
357
}358
359
rq_remove(thread);
360
wq_dequeue(thread);
361
sleep_heap_remove(thread);
362
363
thread->wake_tick = 0;
364
thread_set_state(thread, THREAD_STOPPED);
365
thread->stop_signal = signum;
366
thread->stop_reported = false;
367
368
if (!stopping_current) {
369
thread_unclaim(thread);
370
}371
372
if (thread->user_thread) {
373
sched_thread_t *parent = find_thread(thread->ppid);
374
375
if (parent) {
376
sched_wake_one(&parent->wait_queue);
377
sched_signal_send_thread(parent, SIGCHLD);
378
}379
}380
381
sched_lock_restore(flags);
382
383
if (stopping_current && sched_running_get()) {
384
sched_yield();
385
}386
}387
388
void sched_continue_thread(sched_thread_t *thread) {
389
if (!thread || sched_thread_is_idle(thread)) {
390
return;
391
}392
393
unsigned long flags = sched_lock_save();
394
if (thread_get_state(thread) != THREAD_STOPPED) {
395
sched_lock_restore(flags);
396
return;
397
}398
399
thread_unclaim(thread);
400
thread_set_state(thread, THREAD_READY);
401
thread->stop_signal = 0;
402
thread->stop_reported = false;
403
enqueue_thread(thread);
404
405
sched_lock_restore(flags);
406
}