kernel/
grant.rs

1// Licensed under the Apache License, Version 2.0 or the MIT License.
2// SPDX-License-Identifier: Apache-2.0 OR MIT
3// Copyright Tock Contributors 2022.
4
5//! Support for processes granting memory from their allocations to the kernel.
6//!
7//! ## Grant Overview
8//!
9//! Grants allow capsules to dynamically allocate memory from a process to hold
10//! state on the process's behalf.
11//!
12//! Each capsule that wishes to do this needs to have a [`Grant`] type. Grants
13//! are created at boot, and each have a unique ID and a type `T`. This type
14//! only allows the capsule to allocate memory from a process in the future. It
15//! does not initially represent any allocated memory.
16//!
17//! When a capsule does wish to use its Grant to allocate memory from a process,
18//! it must "enter" the Grant with a specific [`ProcessId`]. Entering a Grant
19//! for a specific process instructs the core kernel to create an object `T` in
20//! the process's memory space and provide the capsule with access to it. If the
21//! Grant has not previously been entered for that process, the memory for
22//! object `T` will be allocated from the "grant region" within the
23//! kernel-accessible portion of the process's memory.
24//!
25//! If a Grant has never been entered for a process, the object `T` will _not_
26//! be allocated in that process's grant region, even if the `Grant` has been
27//! entered for other processes.
28//!
29//! Upcalls and allowed buffer references are stored in the dynamically
30//! allocated grant for a particular Driver as well. Upcalls and allowed buffer
31//! references are stored outside of the `T` object to enable the kernel to
32//! manage them and ensure swapping guarantees are met.
33//!
34//! The type `T` of a Grant is fixed in size and the number of upcalls and
35//! allowed buffers associated with a grant is fixed. That is, when a Grant is
36//! entered for a process the resulting allocated object will be the size of
37//! `SizeOf<T>` plus the size for the structure to hold upcalls and allowed
38//! buffer references. If capsules need additional process-specific memory for
39//! their operation, they can use an [`GrantRegionAllocator`] to request
40//! additional memory from the process's grant region.
41//!
42//! ```text,ignore
43//!                            ┌──────────────────┐
44//!                            │                  │
45//!                            │ Capsule          │
46//!                            │                  │
47//!                            └─┬────────────────┘
48//!                              │ Capsules hold
49//!                              │ references to
50//!                              │ grants.
51//!                              ▼
52//!                            ┌──────────────────┐
53//!                            │ Grant            │
54//!                            │                  │
55//!  Process Memory            │ Type: T          │
56//! ┌────────────────────────┐ │ grant_num: 1     │
57//! │                        │ │ driver_num: 0x4  │
58//! │  ...                   │ └───┬─────────────┬┘
59//! ├────────────────────────┤     │Each Grant   │
60//! │ Grant       ptr 0      │     │has a pointer│
61//! │ Pointers    ptr 1 ───┐ │ ◄───┘per process. │
62//! │             ...      │ │                   │
63//! │             ptr N    │ │                   │
64//! ├──────────────────────┼─┤                   │
65//! │  ...                 │ │                   │
66//! ├──────────────────────┼─┤                   │
67//! │ Grant Region         │ │     When a Grant  │
68//! │                      │ │     is allocated  │
69//! │ ┌─────────────────┐  │ │     for a process │
70//! │ │ Allocated Grant │  │ │ ◄─────────────────┘
71//! │ │                 │  │ │     it uses memory
72//! │ │  [ SizeOf<T> ]  │  │ │     from the grant
73//! │ │─────────────────│  │ │     region.
74//! │ │ Padding         │  │ │
75//! │ │─────────────────│  │ │
76//! │ │ GrantKernelData │  │ │
77//! │ └─────────────────┘◄─┘ │
78//! │                        │
79//! │ ┌─────────────────┐    │
80//! │ │ Custom Grant    │    │ ◄── Capsules can
81//! │ │                 │    │     allocate extra
82//! │ └─────────────────┘    │     memory if needed.
83//! │                        │
84//! ├─kernel_brk─────────────┤
85//! │                        │
86//! │ ...                    │
87//! └────────────────────────┘
88//! ```
89//!
90//! ## Grant Mechanisms and Types
91//!
92//! Here is an overview of the types used by grant.rs to implement the Grant
93//! functionality in Tock:
94//!
95//! ```text,ignore
96//!                         ┌──────────────────────────┐
97//!                         │ struct Grant<T, ...> {   │
98//!                         │   driver_num: usize      │
99//!                         │   grant_num: usize       │
100//!                         │ }                        ├───┐
101//! Entering a Grant for a  └──┬───────────────────────┘   │
102//! process causes the         │                           │
103//! memory for T to be         │ .enter(ProcessId)         │ .enter(ProcessId, fn)
104//! allocated.                 ▼                           │
105//!                         ┌──────────────────────────┐   │ For convenience,
106//! ProcessGrant represents │ struct ProcessGrant<T> { │   │ allocating and getting
107//! a Grant allocated for a │   number: usize          │   │ access to the T object
108//! specific process.       │   process: &Process      │   │ is combined in one
109//!                         │ }                        │   │ .enter() call.
110//! A provided closure      └──┬───────────────────────┘   │
111//! is given access to         │                           │
112//! the underlying memory      │ .enter(fn)                │
113//! where the T is stored.     ▼                           │
114//!                         ┌────────────────────────────┐ │
115//! GrantData wraps the     │ struct GrantData<T>   {    │◄┘
116//! type and provides       │   data: &mut T             │
117//! mutable access.         │ }                          │
118//! GrantKernelData         │ struct GrantKernelData {   │
119//! provides access to      │   upcalls: [SavedUpcall]   │
120//! scheduling upcalls      │   allow_ro: [SavedAllowRo] │
121//! and process buffers.    │   allow_rw: [SavedAllowRW] │
122//!                         │ }                          │
123//!                         └──┬─────────────────────────┘
124//! The actual object T can    │
125//! only be accessed inside    │ fn(mem: &GrantData, kernel_data: &GrantKernelData)
126//! the closure.               ▼
127//! ```
128
129use core::cmp;
130use core::marker::PhantomData;
131use core::mem::{align_of, size_of};
132use core::ops::{Deref, DerefMut};
133use core::ptr::{write, NonNull};
134use core::slice;
135
136use crate::kernel::Kernel;
137use crate::process::ProcessSlot;
138use crate::process::{Error, Process, ProcessCustomGrantIdentifier, ProcessId};
139use crate::processbuffer::{ReadOnlyProcessBuffer, ReadWriteProcessBuffer};
140use crate::processbuffer::{ReadOnlyProcessBufferRef, ReadWriteProcessBufferRef};
141use crate::upcall::{Upcall, UpcallError, UpcallId};
142use crate::utilities::capability_ptr::CapabilityPtr;
143use crate::utilities::machine_register::MachineRegister;
144use crate::ErrorCode;
145
146/// Tracks how many upcalls a grant instance supports automatically.
147pub trait UpcallSize {
148    /// The number of upcalls the grant supports.
149    const COUNT: u8;
150}
151
152/// Specifies how many upcalls a grant instance supports automatically.
153pub struct UpcallCount<const NUM: u8>;
154impl<const NUM: u8> UpcallSize for UpcallCount<NUM> {
155    const COUNT: u8 = NUM;
156}
157
158/// Tracks how many read-only allows a grant instance supports automatically.
159pub trait AllowRoSize {
160    /// The number of read-only allows the grant supports.
161    const COUNT: u8;
162}
163
164/// Specifies how many read-only allows a grant instance supports automatically.
165pub struct AllowRoCount<const NUM: u8>;
166impl<const NUM: u8> AllowRoSize for AllowRoCount<NUM> {
167    const COUNT: u8 = NUM;
168}
169
170/// Tracks how many read-write allows a grant instance supports automatically.
171pub trait AllowRwSize {
172    /// The number of read-write allows the grant supports.
173    const COUNT: u8;
174}
175
176/// Specifies how many read-write allows a grant instance supports
177/// automatically.
178pub struct AllowRwCount<const NUM: u8>;
179impl<const NUM: u8> AllowRwSize for AllowRwCount<NUM> {
180    const COUNT: u8 = NUM;
181}
182
183/// Helper that calculated offsets within the kernel owned memory (i.e. the
184/// non-T part of grant).
185///
186/// Example layout of full grant belonging to a single app and driver:
187///
188/// ```text,ignore
189/// 0x003FFC8  ┌────────────────────────────────────┐
190///            │   T                                |
191/// 0x003FFxx  ├  ───────────────────────── ┐ K     |
192///            │   Padding (ensure T aligns)| e     |
193/// 0x003FF44  ├  ───────────────────────── | r     |
194///            │   SavedAllowRwN            | n     |
195///            │   ...                      | e     | G
196///            │   SavedAllowRw1            | l     | r
197///            │   SavedAllowRw0            |       | a
198/// 0x003FF44  ├  ───────────────────────── | O     | n
199///            │   SavedAllowRoN            | w     | t
200///            │   ...                      | n     |
201///            │   SavedAllowRo1            | e     | M
202///            │   SavedAllowRo0            | d     | e
203/// 0x003FF30  ├  ───────────────────────── |       | m
204///            │   SavedUpcallN             | D     | o
205///            │   ...                      | a     | r
206///            │   SavedUpcall1             | t     | y
207///            │   SavedUpcall0             | a     |
208/// 0x003FF24  ├  ───────────────────────── |       |
209///            │   Counters (usize)         |       |
210/// 0x003FF20  └────────────────────────────────────┘
211/// ```
212///
213/// The counters structure is composed as:
214///
215/// ```text,ignore
216/// 0             1             2             3         bytes
217/// |-------------|-------------|-------------|-------------|
218/// | # Upcalls   | # RO Allows | # RW Allows | [unused]    |
219/// |-------------|-------------|-------------|-------------|
220/// ```
221///
222/// This type is created whenever a grant is entered, and is responsible for
223/// ensuring that the grant is closed when it is no longer used. On `Drop`, we
224/// leave the grant. This protects against calling `grant.enter()` without
225/// calling the corresponding `grant.leave()`, perhaps due to accidentally using
226/// the `?` operator.
227struct EnteredGrantKernelManagedLayout<'a> {
228    /// Leaving a grant is handled through the process implementation, so must
229    /// keep a reference to the relevant process.
230    process: &'a dyn Process,
231    /// The grant number of the entered grant that we want to ensure we leave
232    /// properly.
233    grant_num: usize,
234
235    /// The location of the counters structure for the grant.
236    counters_ptr: *mut usize,
237    /// Pointer to the array of saved upcalls.
238    upcalls_array: *mut SavedUpcall,
239    /// Pointer to the array of saved read-only allows.
240    allow_ro_array: *mut SavedAllowRo,
241    /// Pointer to the array of saved read-write allows.
242    allow_rw_array: *mut SavedAllowRw,
243}
244
245/// Represents the number of the upcall elements in the kernel owned section of
246/// the grant.
247#[derive(Copy, Clone)]
248struct UpcallItems(u8);
249/// Represents the number of the read-only allow elements in the kernel owned
250/// section of the grant.
251#[derive(Copy, Clone)]
252struct AllowRoItems(u8);
253/// Represents the number of the read-write allow elements in the kernel owned
254/// section of the grant.
255#[derive(Copy, Clone)]
256struct AllowRwItems(u8);
257/// Represents the size data (in bytes) T within the grant.
258#[derive(Copy, Clone)]
259struct GrantDataSize(usize);
260/// Represents the alignment of data T within the grant.
261#[derive(Copy, Clone)]
262struct GrantDataAlign(usize);
263
264impl<'a> EnteredGrantKernelManagedLayout<'a> {
265    /// Reads the specified pointer as the base of the kernel owned grant region
266    /// that has previously been initialized.
267    ///
268    /// # Safety
269    ///
270    /// The incoming base pointer must be well aligned and already contain
271    /// initialized data in the expected form. There must not be any other
272    /// `EnteredGrantKernelManagedLayout` for the given `base_ptr` at the same
273    /// time, otherwise multiple mutable references to the same upcall/allow
274    /// slices could be created.
275    unsafe fn read_from_base(
276        base_ptr: NonNull<u8>,
277        process: &'a dyn Process,
278        grant_num: usize,
279    ) -> Self {
280        let counters_ptr = base_ptr.as_ptr() as *mut usize;
281        let counters_val = counters_ptr.read();
282
283        // Parse the counters field for each of the fields
284        let [_, _, allow_ro_num, upcalls_num] = u32::to_be_bytes(counters_val as u32);
285
286        // Skip over the counter usize, then the stored array of `SavedAllowRo`
287        // items and `SavedAllowRw` items.
288        let upcalls_array = counters_ptr.add(1) as *mut SavedUpcall;
289        let allow_ro_array = upcalls_array.add(upcalls_num as usize) as *mut SavedAllowRo;
290        let allow_rw_array = allow_ro_array.add(allow_ro_num as usize) as *mut SavedAllowRw;
291
292        Self {
293            process,
294            grant_num,
295            counters_ptr,
296            upcalls_array,
297            allow_ro_array,
298            allow_rw_array,
299        }
300    }
301
302    /// Creates a layout from the specified pointer and lengths of arrays and
303    /// initializes the kernel owned portion of the layout.
304    ///
305    /// # Safety
306    ///
307    /// The incoming base pointer must be well aligned and reference enough
308    /// memory to hold the entire kernel managed grant structure. There must
309    /// not be any other `EnteredGrantKernelManagedLayout` for
310    /// the given `base_ptr` at the same time, otherwise multiple mutable
311    /// references to the same upcall/allow slices could be created.
312    unsafe fn initialize_from_counts(
313        base_ptr: NonNull<u8>,
314        upcalls_num_val: UpcallItems,
315        allow_ro_num_val: AllowRoItems,
316        allow_rw_num_val: AllowRwItems,
317        process: &'a dyn Process,
318        grant_num: usize,
319    ) -> Self {
320        let counters_ptr = base_ptr.as_ptr() as *mut usize;
321
322        // Create the counters usize value by correctly packing the various
323        // counts into 8 bit fields.
324        let counter: usize =
325            u32::from_be_bytes([0, allow_rw_num_val.0, allow_ro_num_val.0, upcalls_num_val.0])
326                as usize;
327
328        let upcalls_array = counters_ptr.add(1) as *mut SavedUpcall;
329        let allow_ro_array = upcalls_array.add(upcalls_num_val.0.into()) as *mut SavedAllowRo;
330        let allow_rw_array = allow_ro_array.add(allow_ro_num_val.0.into()) as *mut SavedAllowRw;
331
332        counters_ptr.write(counter);
333        write_default_array(upcalls_array, upcalls_num_val.0.into());
334        write_default_array(allow_ro_array, allow_ro_num_val.0.into());
335        write_default_array(allow_rw_array, allow_rw_num_val.0.into());
336
337        Self {
338            process,
339            grant_num,
340            counters_ptr,
341            upcalls_array,
342            allow_ro_array,
343            allow_rw_array,
344        }
345    }
346
347    /// Returns the entire grant size including the kernel owned memory,
348    /// padding, and data for T. Requires that grant_t_align be a power of 2,
349    /// which is guaranteed from align_of rust calls.
350    fn grant_size(
351        upcalls_num: UpcallItems,
352        allow_ro_num: AllowRoItems,
353        allow_rw_num: AllowRwItems,
354        grant_t_size: GrantDataSize,
355        grant_t_align: GrantDataAlign,
356    ) -> usize {
357        let kernel_managed_size = size_of::<usize>()
358            + upcalls_num.0 as usize * size_of::<SavedUpcall>()
359            + allow_ro_num.0 as usize * size_of::<SavedAllowRo>()
360            + allow_rw_num.0 as usize * size_of::<SavedAllowRw>();
361        // We know that grant_t_align is a power of 2, so we can make a mask
362        // that will save only the remainder bits.
363        let grant_t_align_mask = grant_t_align.0 - 1;
364        // Determine padding to get to the next multiple of grant_t_align by
365        // taking the remainder and subtracting that from the alignment, then
366        // ensuring a full alignment value maps to 0.
367        let padding =
368            (grant_t_align.0 - (kernel_managed_size & grant_t_align_mask)) & grant_t_align_mask;
369        kernel_managed_size + padding + grant_t_size.0
370    }
371
372    /// Returns the alignment of the entire grant region based on the alignment
373    /// of data T.
374    fn grant_align(grant_t_align: GrantDataAlign) -> usize {
375        // The kernel owned memory all aligned to usize. We need to use the
376        // higher of the two alignment to ensure our padding calculations work
377        // for any alignment of T.
378        cmp::max(align_of::<usize>(), grant_t_align.0)
379    }
380
381    /// Returns the offset for the grant data t within the entire grant region.
382    ///
383    /// # Safety
384    ///
385    /// The caller must ensure that the specified base pointer is aligned to at
386    /// least the alignment of T and points to a grant that is of size
387    /// grant_size bytes.
388    unsafe fn offset_of_grant_data_t(
389        base_ptr: NonNull<u8>,
390        grant_size: usize,
391        grant_t_size: GrantDataSize,
392    ) -> NonNull<u8> {
393        // The location of the grant data T is the last element in the entire
394        // grant region. Caller must verify that memory is accessible and well
395        // aligned to T.
396        let grant_t_size_usize: usize = grant_t_size.0;
397        NonNull::new_unchecked(base_ptr.as_ptr().add(grant_size - grant_t_size_usize))
398    }
399
400    /// Read an 8 bit value from the counter field offset by the specified
401    /// number of bits. This is a helper function for reading the counter field.
402    fn get_counter_offset(&self, offset_bits: usize) -> usize {
403        // # Safety
404        //
405        // Creating a `EnteredGrantKernelManagedLayout` object requires that the
406        // pointers are well aligned and point to valid memory.
407        let counters_val = unsafe { self.counters_ptr.read() };
408        (counters_val >> offset_bits) & 0xFF
409    }
410
411    /// Return the number of upcalls stored by the core kernel for this grant.
412    fn get_upcalls_number(&self) -> usize {
413        self.get_counter_offset(0)
414    }
415
416    /// Return the number of read-only allow buffers stored by the core kernel
417    /// for this grant.
418    fn get_allow_ro_number(&self) -> usize {
419        self.get_counter_offset(8)
420    }
421
422    /// Return the number of read-write allow buffers stored by the core kernel
423    /// for this grant.
424    fn get_allow_rw_number(&self) -> usize {
425        self.get_counter_offset(16)
426    }
427
428    /// Return mutable access to the slice of stored upcalls for this grant.
429    /// This is necessary for storing a new upcall.
430    fn get_upcalls_slice(&mut self) -> &mut [SavedUpcall] {
431        // # Safety
432        //
433        // Creating a `EnteredGrantKernelManagedLayout` object ensures that the
434        // pointer to the upcall array is valid.
435        unsafe { slice::from_raw_parts_mut(self.upcalls_array, self.get_upcalls_number()) }
436    }
437
438    /// Return mutable access to the slice of stored read-only allow buffers for
439    /// this grant. This is necessary for storing a new read-only allow.
440    fn get_allow_ro_slice(&mut self) -> &mut [SavedAllowRo] {
441        // # Safety
442        //
443        // Creating a `EnteredGrantKernelManagedLayout` object ensures that the
444        // pointer to the RO allow array is valid.
445        unsafe { slice::from_raw_parts_mut(self.allow_ro_array, self.get_allow_ro_number()) }
446    }
447
448    /// Return mutable access to the slice of stored read-write allow buffers
449    /// for this grant. This is necessary for storing a new read-write allow.
450    fn get_allow_rw_slice(&mut self) -> &mut [SavedAllowRw] {
451        // # Safety
452        //
453        // Creating a `EnteredGrantKernelManagedLayout` object ensures that the
454        // pointer to the RW allow array is valid.
455        unsafe { slice::from_raw_parts_mut(self.allow_rw_array, self.get_allow_rw_number()) }
456    }
457
458    /// Return slices to the kernel managed upcalls and allow buffers. This
459    /// permits using upcalls and allow buffers when a capsule is accessing a
460    /// grant.
461    fn get_resource_slices(&self) -> (&[SavedUpcall], &[SavedAllowRo], &[SavedAllowRw]) {
462        // # Safety
463        //
464        // Creating a `EnteredGrantKernelManagedLayout` object ensures that the
465        // pointer to the upcall array is valid.
466        let upcall_slice =
467            unsafe { slice::from_raw_parts(self.upcalls_array, self.get_upcalls_number()) };
468
469        // # Safety
470        //
471        // Creating a `EnteredGrantKernelManagedLayout` object ensures that the
472        // pointer to the RO allow array is valid.
473        let allow_ro_slice =
474            unsafe { slice::from_raw_parts(self.allow_ro_array, self.get_allow_ro_number()) };
475
476        // # Safety
477        //
478        // Creating a `KernelManagedLayout` object ensures that the pointer to
479        // the RW allow array is valid.
480        let allow_rw_slice =
481            unsafe { slice::from_raw_parts(self.allow_rw_array, self.get_allow_rw_number()) };
482
483        (upcall_slice, allow_ro_slice, allow_rw_slice)
484    }
485}
486
487// Ensure that we leave the grant once this goes out of scope.
488impl Drop for EnteredGrantKernelManagedLayout<'_> {
489    fn drop(&mut self) {
490        // ### Safety
491        //
492        // To safely call this function we must ensure that no references will
493        // exist to the grant once `leave_grant()` returns. Because using a
494        // `EnteredGrantKernelManagedLayout` object is the only only way we
495        // access the actual memory of a grant, and we are calling
496        // `leave_grant()` from its `drop()` method, we are sure there will be
497        // no remaining references to the grant.
498        unsafe {
499            self.process.leave_grant(self.grant_num);
500        }
501    }
502}
503
504/// This [`GrantData`] object provides access to the memory allocated for a
505/// grant for a specific process.
506///
507/// The [`GrantData`] type is templated on `T`, the actual type of the object in
508/// the grant. [`GrantData'] holds a mutable reference to the type, allowing
509/// users access to the object in process memory.
510///
511/// Capsules gain access to a [`GrantData`] object by calling
512/// [`Grant::enter()`].
513pub struct GrantData<'a, T: 'a + ?Sized> {
514    /// The mutable reference to the actual object type stored in the grant.
515    data: &'a mut T,
516}
517
518impl<'a, T: 'a + ?Sized> GrantData<'a, T> {
519    /// Create a [`GrantData`] object to provide access to the actual object
520    /// allocated for a process.
521    ///
522    /// Only one can [`GrantData`] per underlying object can be created at a
523    /// time. Otherwise, there would be multiple mutable references to the same
524    /// object which is undefined behavior.
525    fn new(data: &'a mut T) -> GrantData<'a, T> {
526        GrantData { data }
527    }
528}
529
530impl<'a, T: 'a + ?Sized> Deref for GrantData<'a, T> {
531    type Target = T;
532    fn deref(&self) -> &T {
533        self.data
534    }
535}
536
537impl<'a, T: 'a + ?Sized> DerefMut for GrantData<'a, T> {
538    fn deref_mut(&mut self) -> &mut T {
539        self.data
540    }
541}
542
543/// This [`GrantKernelData`] object provides a handle to access upcalls and
544/// process buffers stored on behalf of a particular grant/driver.
545///
546/// Capsules gain access to a [`GrantKernelData`] object by calling
547/// [`Grant::enter()`]. From there, they can schedule upcalls or access process
548/// buffers.
549///
550/// It is expected that this type will only exist as a short-lived stack
551/// allocation, so its size is not a significant concern.
552pub struct GrantKernelData<'a> {
553    /// A reference to the actual upcall slice stored in the grant.
554    upcalls: &'a [SavedUpcall],
555
556    /// A reference to the actual read only allow slice stored in the grant.
557    allow_ro: &'a [SavedAllowRo],
558
559    /// A reference to the actual read write allow slice stored in the grant.
560    allow_rw: &'a [SavedAllowRw],
561
562    /// We need to keep track of the driver number so we can properly identify
563    /// the Upcall that is called. We need to keep track of its source so we can
564    /// remove it if the Upcall is unsubscribed.
565    driver_num: usize,
566
567    /// A reference to the process that these upcalls are for. This is used for
568    /// actually scheduling the upcalls.
569    process: &'a dyn Process,
570}
571
572impl<'a> GrantKernelData<'a> {
573    /// Create a [`GrantKernelData`] object to provide a handle for capsules to
574    /// call Upcalls.
575    fn new(
576        upcalls: &'a [SavedUpcall],
577        allow_ro: &'a [SavedAllowRo],
578        allow_rw: &'a [SavedAllowRw],
579        driver_num: usize,
580        process: &'a dyn Process,
581    ) -> GrantKernelData<'a> {
582        Self {
583            upcalls,
584            allow_ro,
585            allow_rw,
586            driver_num,
587            process,
588        }
589    }
590
591    /// Schedule the specified upcall for the process with r0, r1, r2 as
592    /// provided values.
593    ///
594    /// Capsules call this function to schedule upcalls, and upcalls are
595    /// identified by the `subscribe_num`, which must match the subscribe number
596    /// used when the upcall was originally subscribed by a process.
597    /// `subscribe_num`s are indexed starting at zero.
598    pub fn schedule_upcall(
599        &self,
600        subscribe_num: usize,
601        r: (usize, usize, usize),
602    ) -> Result<(), UpcallError> {
603        // Implement `self.upcalls[subscribe_num]` without a chance of a panic.
604        self.upcalls.get(subscribe_num).map_or(
605            Err(UpcallError::InvalidSubscribeNum),
606            |saved_upcall| {
607                // We can create an `Upcall` object based on what is stored in
608                // the process grant and use that to add the upcall to the
609                // pending array for the process.
610                let upcall = Upcall::new(
611                    self.process.processid(),
612                    UpcallId {
613                        subscribe_num,
614                        driver_num: self.driver_num,
615                    },
616                    saved_upcall.appdata,
617                    saved_upcall.fn_ptr,
618                );
619                upcall.schedule(self.process, r.0, r.1, r.2)
620            },
621        )
622    }
623
624    /// Search the work queue for the first pending operation with the given
625    /// `subscribe_num` and if one exists remove it from the task queue.
626    ///
627    /// Returns the associated [`Task`](crate::process::Task) if one was found, otherwise returns
628    /// [`None`].
629    pub fn remove_upcall(&self, subscribe_num: usize) -> Option<crate::process::Task> {
630        self.process.remove_upcall(UpcallId {
631            subscribe_num,
632            driver_num: self.driver_num,
633        })
634    }
635
636    /// Remove all scheduled upcalls with the given `subscribe_num` from the
637    /// task queue.
638    ///
639    /// Returns the number of removed upcalls.
640    pub fn remove_pending_upcalls(&self, subscribe_num: usize) -> usize {
641        self.process.remove_pending_upcalls(UpcallId {
642            subscribe_num,
643            driver_num: self.driver_num,
644        })
645    }
646
647    /// Returns a lifetime limited reference to the requested
648    /// [`ReadOnlyProcessBuffer`].
649    ///
650    /// The len of the returned [`ReadOnlyProcessBuffer`] must be checked by the
651    /// caller to ensure that a buffer has in fact been allocated. An
652    /// unallocated buffer will be returned as a [`ReadOnlyProcessBuffer`] of
653    /// length 0.
654    ///
655    /// The [`ReadOnlyProcessBuffer`] is only valid for as long as this object
656    /// is valid, i.e. the lifetime of the app enter closure.
657    ///
658    /// If the specified allow number is invalid, then a
659    /// [`crate::process::Error::AddressOutOfBounds`] will be returned. This
660    /// returns a [`crate::process::Error`] to allow for easy chaining of this
661    /// function with the `ReadOnlyProcessBuffer::enter()` function with
662    /// `and_then`.
663    pub fn get_readonly_processbuffer(
664        &self,
665        allow_ro_num: usize,
666    ) -> Result<ReadOnlyProcessBufferRef, crate::process::Error> {
667        self.allow_ro.get(allow_ro_num).map_or(
668            Err(crate::process::Error::AddressOutOfBounds),
669            |saved_ro| {
670                // # Safety
671                //
672                // This is the saved process buffer data has been validated to
673                // be wholly contained within this process before it was stored.
674                // The lifetime of the ReadOnlyProcessBuffer is bound to the
675                // lifetime of self, which correctly limits dereferencing this
676                // saved pointer to only when it is valid.
677                unsafe {
678                    Ok(ReadOnlyProcessBufferRef::new(
679                        saved_ro.ptr,
680                        saved_ro.len,
681                        self.process.processid(),
682                    ))
683                }
684            },
685        )
686    }
687
688    /// Returns a lifetime limited reference to the requested
689    /// [`ReadWriteProcessBuffer`].
690    ///
691    /// The length of the returned [`ReadWriteProcessBuffer`] must be checked by
692    /// the caller to ensure that a buffer has in fact been allocated. An
693    /// unallocated buffer will be returned as a [`ReadWriteProcessBuffer`] of
694    /// length 0.
695    ///
696    /// The [`ReadWriteProcessBuffer`] is only value for as long as this object
697    /// is valid, i.e. the lifetime of the app enter closure.
698    ///
699    /// If the specified allow number is invalid, then a
700    /// [`crate::process::Error::AddressOutOfBounds`] will be returned. This
701    /// returns a [`crate::process::Error`] to allow for easy chaining of this
702    /// function with the `ReadWriteProcessBuffer::enter()` function with
703    /// `and_then`.
704    pub fn get_readwrite_processbuffer(
705        &self,
706        allow_rw_num: usize,
707    ) -> Result<ReadWriteProcessBufferRef, crate::process::Error> {
708        self.allow_rw.get(allow_rw_num).map_or(
709            Err(crate::process::Error::AddressOutOfBounds),
710            |saved_rw| {
711                // # Safety
712                //
713                // This is the saved process buffer data has been validated to
714                // be wholly contained within this process before it was stored.
715                // The lifetime of the ReadWriteProcessBuffer is bound to the
716                // lifetime of self, which correctly limits dereferencing this
717                // saved pointer to only when it is valid.
718                unsafe {
719                    Ok(ReadWriteProcessBufferRef::new(
720                        saved_rw.ptr,
721                        saved_rw.len,
722                        self.process.processid(),
723                    ))
724                }
725            },
726        )
727    }
728}
729
730/// A minimal representation of an upcall, used for storing an upcall in a
731/// process' grant table without wasting memory duplicating information such as
732/// process ID.
733#[repr(C)]
734#[derive(Default)]
735struct SavedUpcall {
736    appdata: MachineRegister,
737    fn_ptr: CapabilityPtr,
738}
739
740/// A minimal representation of a read-only allow from app, used for storing a
741/// read-only allow in a process' kernel managed grant space without wasting
742/// memory duplicating information such as process ID.
743#[repr(C)]
744struct SavedAllowRo {
745    ptr: *const u8,
746    len: usize,
747}
748
749// This allow is still needed on the current stable compiler, but generates a warning
750// on the current nightly compiler, as of 05/18/2025. So allow this warning for now.
751// This can probably be fixed on the next nightly update.
752#[allow(clippy::derivable_impls)]
753impl Default for SavedAllowRo {
754    fn default() -> Self {
755        Self {
756            ptr: core::ptr::null(),
757            len: 0,
758        }
759    }
760}
761
762/// A minimal representation of a read-write allow from app, used for storing a
763/// read-write allow in a process' kernel managed grant space without wasting
764/// memory duplicating information such as process ID.
765#[repr(C)]
766struct SavedAllowRw {
767    ptr: *mut u8,
768    len: usize,
769}
770
771// This allow is still needed on the current stable compiler, but generates a warning
772// on the current nightly compiler, as of 05/18/2025. So allow this warning for now.
773// This can probably be fixed on the next nightly update.
774#[allow(clippy::derivable_impls)]
775impl Default for SavedAllowRw {
776    fn default() -> Self {
777        Self {
778            ptr: core::ptr::null_mut(),
779            len: 0,
780        }
781    }
782}
783
784/// Write the default value of T to every element of the array.
785///
786/// # Safety
787///
788/// The pointer must be well aligned and point to allocated memory that is
789/// writable for `size_of::<T> * num` bytes. No Rust references may exist to
790/// memory in the address range spanned by `base..base+num` at the time this
791/// function is called. The memory does not need to be initialized yet. If it
792/// already does contain initialized memory, then those contents will be
793/// overwritten without being `Drop`ed first.
794unsafe fn write_default_array<T: Default>(base: *mut T, num: usize) {
795    for i in 0..num {
796        base.add(i).write(T::default());
797    }
798}
799
800/// Enters the grant for the specified process. Caller must hold on to the grant
801/// lifetime guard while they accessing the memory in the layout (second
802/// element).
803fn enter_grant_kernel_managed(
804    process: &dyn Process,
805    driver_num: usize,
806) -> Result<EnteredGrantKernelManagedLayout, ErrorCode> {
807    let grant_num = process.lookup_grant_from_driver_num(driver_num)?;
808
809    // Check if the grant has been allocated, and if not we cannot enter this
810    // grant.
811    match process.grant_is_allocated(grant_num) {
812        Some(true) => { /* Allocated, nothing to do */ }
813        Some(false) => return Err(ErrorCode::NOMEM),
814        None => return Err(ErrorCode::FAIL),
815    }
816
817    // Return early if no grant.
818    let grant_base_ptr = process.enter_grant(grant_num).or(Err(ErrorCode::NOMEM))?;
819    // # Safety
820    //
821    // We know that this pointer is well aligned and initialized with meaningful
822    // data when the grant region was allocated.
823    let layout = unsafe {
824        EnteredGrantKernelManagedLayout::read_from_base(grant_base_ptr, process, grant_num)
825    };
826    Ok(layout)
827}
828
829/// Subscribe to an upcall by saving the upcall in the grant region for the
830/// process and returning the existing upcall for the same UpcallId.
831pub(crate) fn subscribe(
832    process: &dyn Process,
833    upcall: Upcall,
834) -> Result<Upcall, (Upcall, ErrorCode)> {
835    // Enter grant and keep it open until _grant_open goes out of scope.
836    let mut layout = match enter_grant_kernel_managed(process, upcall.upcall_id.driver_num) {
837        Ok(val) => val,
838        Err(e) => return Err((upcall, e)),
839    };
840
841    // Create the saved upcalls slice from the grant memory.
842    //
843    // # Safety
844    //
845    // This is safe because of how the grant was initially allocated and that
846    // because we were able to enter the grant the grant region must be valid
847    // and initialized. We are also holding the grant open until `_grant_open`
848    // goes out of scope.
849    let saved_upcalls_slice = layout.get_upcalls_slice();
850
851    // Index into the saved upcall slice to get the old upcall. Use .get in case
852    // userspace passed us a bad subscribe number.
853    match saved_upcalls_slice.get_mut(upcall.upcall_id.subscribe_num) {
854        Some(saved_upcall) => {
855            // Create an `Upcall` object with the old saved upcall.
856            let old_upcall = Upcall::new(
857                process.processid(),
858                upcall.upcall_id,
859                saved_upcall.appdata,
860                saved_upcall.fn_ptr,
861            );
862
863            // Overwrite the saved upcall with the new upcall.
864            saved_upcall.appdata = upcall.appdata;
865            saved_upcall.fn_ptr = upcall.fn_ptr;
866
867            // Success!
868            Ok(old_upcall)
869        }
870        None => Err((upcall, ErrorCode::NOSUPPORT)),
871    }
872}
873
874/// Stores the specified read-only process buffer in the kernel managed grant
875/// region for this process and driver. The previous read-only process buffer
876/// stored at the same allow_num id is returned.
877pub(crate) fn allow_ro(
878    process: &dyn Process,
879    driver_num: usize,
880    allow_num: usize,
881    buffer: ReadOnlyProcessBuffer,
882) -> Result<ReadOnlyProcessBuffer, (ReadOnlyProcessBuffer, ErrorCode)> {
883    // Enter grant and keep it open until `_grant_open` goes out of scope.
884    let mut layout = match enter_grant_kernel_managed(process, driver_num) {
885        Ok(val) => val,
886        Err(e) => return Err((buffer, e)),
887    };
888
889    // Create the saved allow ro slice from the grant memory.
890    //
891    // # Safety
892    //
893    // This is safe because of how the grant was initially allocated and that
894    // because we were able to enter the grant the grant region must be valid
895    // and initialized. We are also holding the grant open until _grant_open
896    // goes out of scope.
897    let saved_allow_ro_slice = layout.get_allow_ro_slice();
898
899    // Index into the saved slice to get the old value. Use .get in case
900    // userspace passed us a bad allow number.
901    match saved_allow_ro_slice.get_mut(allow_num) {
902        Some(saved) => {
903            // # Safety
904            //
905            // The pointer has already been validated to be within application
906            // memory before storing the values in the saved slice.
907            let old_allow =
908                unsafe { ReadOnlyProcessBuffer::new(saved.ptr, saved.len, process.processid()) };
909
910            // Replace old values with current buffer.
911            let (ptr, len) = buffer.consume();
912            saved.ptr = ptr;
913            saved.len = len;
914
915            // Success!
916            Ok(old_allow)
917        }
918        None => Err((buffer, ErrorCode::NOSUPPORT)),
919    }
920}
921
922/// Stores the specified read-write process buffer in the kernel managed grant
923/// region for this process and driver. The previous read-write process buffer
924/// stored at the same allow_num id is returned.
925pub(crate) fn allow_rw(
926    process: &dyn Process,
927    driver_num: usize,
928    allow_num: usize,
929    buffer: ReadWriteProcessBuffer,
930) -> Result<ReadWriteProcessBuffer, (ReadWriteProcessBuffer, ErrorCode)> {
931    // Enter grant and keep it open until `_grant_open` goes out of scope.
932    let mut layout = match enter_grant_kernel_managed(process, driver_num) {
933        Ok(val) => val,
934        Err(e) => return Err((buffer, e)),
935    };
936
937    // Create the saved allow rw slice from the grant memory.
938    //
939    // # Safety
940    //
941    // This is safe because of how the grant was initially allocated and that
942    // because we were able to enter the grant the grant region must be valid
943    // and initialized. We are also holding the grant open until `_grant_open`
944    // goes out of scope.
945    let saved_allow_rw_slice = layout.get_allow_rw_slice();
946
947    // Index into the saved slice to get the old value. Use .get in case
948    // userspace passed us a bad allow number.
949    match saved_allow_rw_slice.get_mut(allow_num) {
950        Some(saved) => {
951            // # Safety
952            //
953            // The pointer has already been validated to be within application
954            // memory before storing the values in the saved slice.
955            let old_allow =
956                unsafe { ReadWriteProcessBuffer::new(saved.ptr, saved.len, process.processid()) };
957
958            // Replace old values with current buffer.
959            let (ptr, len) = buffer.consume();
960            saved.ptr = ptr;
961            saved.len = len;
962
963            // Success!
964            Ok(old_allow)
965        }
966        None => Err((buffer, ErrorCode::NOSUPPORT)),
967    }
968}
969
970/// An instance of a grant allocated for a particular process.
971///
972/// [`ProcessGrant`] is a handle to an instance of a grant that has been
973/// allocated in a specific process's grant region. A [`ProcessGrant`]
974/// guarantees that the memory for the grant has been allocated in the process's
975/// memory.
976///
977/// This is created from a [`Grant`] when that grant is entered for a specific
978/// process.
979pub struct ProcessGrant<
980    'a,
981    T: 'a,
982    Upcalls: UpcallSize,
983    AllowROs: AllowRoSize,
984    AllowRWs: AllowRwSize,
985> {
986    /// The process the grant is applied to.
987    ///
988    /// We use a reference here because instances of [`ProcessGrant`] are very
989    /// short lived. They only exist while a [`Grant`] is being entered, so we
990    /// can be sure the process still exists while a `ProcessGrant` exists. No
991    /// [`ProcessGrant`] can be stored.
992    process: &'a dyn Process,
993
994    /// The syscall driver number this grant is associated with.
995    driver_num: usize,
996
997    /// The identifier of the Grant this is applied for.
998    grant_num: usize,
999
1000    /// Used to store Rust types for grant.
1001    _phantom: PhantomData<(T, Upcalls, AllowROs, AllowRWs)>,
1002}
1003
1004impl<'a, T: Default, Upcalls: UpcallSize, AllowROs: AllowRoSize, AllowRWs: AllowRwSize>
1005    ProcessGrant<'a, T, Upcalls, AllowROs, AllowRWs>
1006{
1007    /// Create a [`ProcessGrant`] for the given Grant in the given Process's
1008    /// grant region.
1009    ///
1010    /// If the grant in this process has not been setup before this will attempt
1011    /// to allocate the memory from the process's grant region.
1012    ///
1013    /// # Return
1014    ///
1015    /// If the grant is already allocated or could be allocated, and the process
1016    /// is valid, this returns `Ok(ProcessGrant)`. Otherwise it returns a
1017    /// relevant error.
1018    fn new(
1019        grant: &Grant<T, Upcalls, AllowROs, AllowRWs>,
1020        processid: ProcessId,
1021    ) -> Result<Self, Error> {
1022        // Moves non-generic code from new() into non-generic function to reduce
1023        // code bloat from the generic function being monomorphized, as it is
1024        // common to have over 50 copies of Grant::enter() in a Tock kernel (and
1025        // thus 50+ copies of this function). The returned Option indicates if
1026        // the returned pointer still needs to be initialized (in the case where
1027        // the grant was only just allocated).
1028        fn new_inner<'a>(
1029            grant_num: usize,
1030            driver_num: usize,
1031            grant_t_size: GrantDataSize,
1032            grant_t_align: GrantDataAlign,
1033            num_upcalls: UpcallItems,
1034            num_allow_ros: AllowRoItems,
1035            num_allow_rws: AllowRwItems,
1036            processid: ProcessId,
1037        ) -> Result<(Option<NonNull<u8>>, &'a dyn Process), Error> {
1038            // Here is an example of how the grants are laid out in the grant
1039            // region of process's memory:
1040            //
1041            // Mem. Addr.
1042            // 0x0040000  ┌────────────────────────────────────
1043            //            │   DriverNumN    [0x1]
1044            //            │   GrantPointerN [0x003FFC8]
1045            //            │   ...
1046            //            │   DriverNum1    [0x60000]
1047            //            │   GrantPointer1 [0x003FFC0]
1048            //            │   DriverNum0
1049            //            │   GrantPointer0 [0x0000000 (NULL)]
1050            //            ├────────────────────────────────────
1051            //            │   Process Control Block
1052            // 0x003FFE0  ├────────────────────────────────────  Grant Region ┐
1053            //            │   GrantDataN                                      │
1054            // 0x003FFC8  ├────────────────────────────────────               │
1055            //            │   GrantData1                                      ▼
1056            // 0x003FF20  ├────────────────────────────────────
1057            //            │
1058            //            │   --unallocated--
1059            //            │
1060            //            └────────────────────────────────────
1061            //
1062            // An array of pointers (one per possible grant region) point to
1063            // where the actual grant memory is allocated inside of the process.
1064            // The grant memory is not allocated until the actual grant region
1065            // is actually used.
1066
1067            let process = processid
1068                .kernel
1069                .get_process(processid)
1070                .ok_or(Error::NoSuchApp)?;
1071
1072            // Check if the grant is allocated. If not, we allocate it process
1073            // memory first. We then create an `ProcessGrant` object for this
1074            // grant.
1075            if let Some(is_allocated) = process.grant_is_allocated(grant_num) {
1076                if !is_allocated {
1077                    // Calculate the alignment and size for entire grant region.
1078                    let alloc_align = EnteredGrantKernelManagedLayout::grant_align(grant_t_align);
1079                    let alloc_size = EnteredGrantKernelManagedLayout::grant_size(
1080                        num_upcalls,
1081                        num_allow_ros,
1082                        num_allow_rws,
1083                        grant_t_size,
1084                        grant_t_align,
1085                    );
1086
1087                    // Allocate grant, the memory is still uninitialized though.
1088                    if process
1089                        .allocate_grant(grant_num, driver_num, alloc_size, alloc_align)
1090                        .is_err()
1091                    {
1092                        return Err(Error::OutOfMemory);
1093                    }
1094
1095                    let grant_ptr = process.enter_grant(grant_num)?;
1096
1097                    // Create a layout from the counts we have and initialize
1098                    // all memory so it is valid in the future to read as a
1099                    // reference.
1100                    //
1101                    // # Safety
1102                    //
1103                    // - The grant base pointer is well aligned, yet does not
1104                    //   have initialized data.
1105                    // - The pointer points to a large enough space to correctly
1106                    //   write to is guaranteed by alloc of size
1107                    //   `EnteredGrantKernelManagedLayout::grant_size`.
1108                    // - There are no proper rust references that map to these
1109                    //   addresses.
1110                    unsafe {
1111                        let _layout = EnteredGrantKernelManagedLayout::initialize_from_counts(
1112                            grant_ptr,
1113                            num_upcalls,
1114                            num_allow_ros,
1115                            num_allow_rws,
1116                            process,
1117                            grant_num,
1118                        );
1119                    }
1120
1121                    // # Safety
1122                    //
1123                    // The grant pointer points to an alloc that is alloc_size
1124                    // large and is at least as aligned as grant_t_align.
1125                    unsafe {
1126                        Ok((
1127                            Some(EnteredGrantKernelManagedLayout::offset_of_grant_data_t(
1128                                grant_ptr,
1129                                alloc_size,
1130                                grant_t_size,
1131                            )),
1132                            process,
1133                        ))
1134                    }
1135                } else {
1136                    // T was already allocated, outer function should not
1137                    // initialize T.
1138                    Ok((None, process))
1139                }
1140            } else {
1141                // Cannot use the grant region in any way if the process is
1142                // inactive.
1143                Err(Error::InactiveApp)
1144            }
1145        }
1146
1147        // Handle the bulk of the work in a function which is not templated.
1148        let (opt_raw_grant_ptr_nn, process) = new_inner(
1149            grant.grant_num,
1150            grant.driver_num,
1151            GrantDataSize(size_of::<T>()),
1152            GrantDataAlign(align_of::<T>()),
1153            UpcallItems(Upcalls::COUNT),
1154            AllowRoItems(AllowROs::COUNT),
1155            AllowRwItems(AllowRWs::COUNT),
1156            processid,
1157        )?;
1158
1159        // We can now do the initialization of T object if necessary.
1160        if let Some(allocated_ptr) = opt_raw_grant_ptr_nn {
1161            // Grant type T
1162            //
1163            // # Safety
1164            //
1165            // This is safe because:
1166            //
1167            // 1. The pointer address is valid. The pointer is allocated
1168            //    statically in process memory, and will exist for as long
1169            //    as the process does. The grant is only accessible while
1170            //    the process is still valid.
1171            //
1172            // 2. The pointer is correctly aligned. The newly allocated
1173            //    grant is aligned for type T, and there is padding inserted
1174            //    between the upcall array and the T object such that the T
1175            //    object starts a multiple of `align_of<T>` from the
1176            //    beginning of the allocation.
1177            unsafe {
1178                // Convert untyped `*mut u8` allocation to allocated type.
1179                let new_region = NonNull::cast::<T>(allocated_ptr);
1180                // We use `ptr::write` to avoid `Drop`ping the uninitialized
1181                // memory in case `T` implements the `Drop` trait.
1182                write(new_region.as_ptr(), T::default());
1183            }
1184        }
1185
1186        // We have ensured the grant is already allocated or was just allocated,
1187        // so we can create and return the `ProcessGrant` type.
1188        Ok(ProcessGrant {
1189            process,
1190            driver_num: grant.driver_num,
1191            grant_num: grant.grant_num,
1192            _phantom: PhantomData,
1193        })
1194    }
1195
1196    /// Return a [`ProcessGrant`] for a grant in a process if the process is
1197    /// valid and that process grant has already been allocated, or `None`
1198    /// otherwise.
1199    fn new_if_allocated(
1200        grant: &Grant<T, Upcalls, AllowROs, AllowRWs>,
1201        process: &'a dyn Process,
1202    ) -> Option<Self> {
1203        if let Some(is_allocated) = process.grant_is_allocated(grant.grant_num) {
1204            if is_allocated {
1205                Some(ProcessGrant {
1206                    process,
1207                    driver_num: grant.driver_num,
1208                    grant_num: grant.grant_num,
1209                    _phantom: PhantomData,
1210                })
1211            } else {
1212                // Grant has not been allocated.
1213                None
1214            }
1215        } else {
1216            // Process is invalid.
1217            None
1218        }
1219    }
1220
1221    /// Return the [`ProcessId`] of the process this [`ProcessGrant`] is
1222    /// associated with.
1223    pub fn processid(&self) -> ProcessId {
1224        self.process.processid()
1225    }
1226
1227    /// Run a function with access to the memory in the related process for the
1228    /// related Grant. This also provides access to any associated Upcalls and
1229    /// allowed buffers stored with the grant.
1230    ///
1231    /// This is "entering" the grant region, and the _only_ time when the
1232    /// contents of a grant region can be accessed.
1233    ///
1234    /// Note, a grant can only be entered once at a time. Attempting to call
1235    /// `.enter()` on a grant while it is already entered will result in a
1236    /// `panic!()`. See the comment in `access_grant()` for more information.
1237    pub fn enter<F, R>(self, fun: F) -> R
1238    where
1239        F: FnOnce(&mut GrantData<T>, &GrantKernelData) -> R,
1240    {
1241        // # `unwrap()` Safety
1242        //
1243        // `access_grant()` can only return `None` if the grant is already
1244        // entered. Since we are asking for a panic!() if the grant is entered,
1245        // `access_grant()` function will never return `None`.
1246        self.access_grant(fun, true).unwrap()
1247    }
1248
1249    /// Run a function with access to the data in the related process for the
1250    /// related Grant only if that grant region is not already entered. If the
1251    /// grant is already entered silently skip it. Also provide access to
1252    /// associated Upcalls.
1253    ///
1254    /// **You almost certainly should use `.enter()` rather than
1255    /// `.try_enter()`.**
1256    ///
1257    /// While the `.enter()` version can panic, that panic likely indicates a
1258    /// bug in the code and not a condition that should be handled. For example,
1259    /// this benign looking code is wrong:
1260    ///
1261    /// ```ignore
1262    /// self.apps.enter(thisapp, |app_grant, _| {
1263    ///     // Update state in the grant region of `thisapp`. Also, mark that
1264    ///     // `thisapp` needs to run again.
1265    ///     app_grant.runnable = true;
1266    ///
1267    ///     // Now, check all apps to see if any are ready to run.
1268    ///     let mut work_left_to_do = false;
1269    ///     self.apps.iter().each(|other_app| {
1270    ///         other_app.enter(|other_app_grant, _| { // ERROR! This leads to a
1271    ///             if other_app_grant.runnable {      // grant being entered
1272    ///                 work_left_to_do = true;        // twice!
1273    ///             }
1274    ///         })
1275    ///     })
1276    /// })
1277    /// ```
1278    ///
1279    /// The example is wrong because it tries to iterate across all grant
1280    /// regions while one of them is already entered. This will lead to a grant
1281    /// region being entered twice which violates Rust's memory restrictions and
1282    /// is undefined behavior.
1283    ///
1284    /// However, since the example uses `.enter()` on the iteration, Tock will
1285    /// panic when the grant is entered for the second time, notifying the
1286    /// developer that something is wrong. The fix is to exit out of the first
1287    /// `.enter()` before attempting to iterate over the grant for all
1288    /// processes.
1289    ///
1290    /// However, if the example used `.try_enter()` in the iter loop, there
1291    /// would be no panic, but the already entered grant would be silently
1292    /// skipped. This can hide subtle bugs if the skipped grant is only relevant
1293    /// in certain cases.
1294    ///
1295    /// Therefore, only use `try_enter()` if you are sure you want to skip the
1296    /// already entered grant. Cases for this are rare.
1297    ///
1298    /// ## Return
1299    ///
1300    /// Returns `None` if the grant is already entered. Otherwise returns
1301    /// `Some(fun())`.
1302    pub fn try_enter<F, R>(self, fun: F) -> Option<R>
1303    where
1304        F: FnOnce(&mut GrantData<T>, &GrantKernelData) -> R,
1305    {
1306        self.access_grant(fun, false)
1307    }
1308
1309    /// Run a function with access to the memory in the related process for the
1310    /// related Grant. Also provide this function with access to any associated
1311    /// Upcalls and an allocator for allocating additional memory in the
1312    /// process's grant region.
1313    ///
1314    /// This is "entering" the grant region, and the _only_ time when the
1315    /// contents of a grant region can be accessed.
1316    ///
1317    /// Note, a grant can only be entered once at a time. Attempting to call
1318    /// `.enter()` on a grant while it is already entered will result in a
1319    /// panic!()`. See the comment in `access_grant()` for more information.
1320    pub fn enter_with_allocator<F, R>(self, fun: F) -> R
1321    where
1322        F: FnOnce(&mut GrantData<T>, &GrantKernelData, &mut GrantRegionAllocator) -> R,
1323    {
1324        // # `unwrap()` Safety
1325        //
1326        // `access_grant()` can only return `None` if the grant is already
1327        // entered. Since we are asking for a panic!() if the grant is entered,
1328        // `access_grant()` function will never return `None`.
1329        self.access_grant_with_allocator(fun, true).unwrap()
1330    }
1331
1332    /// Access the [`ProcessGrant`] memory and run a closure on the process's
1333    /// grant memory.
1334    ///
1335    /// If `panic_on_reenter` is `true`, this will panic if the grant region is
1336    /// already currently entered. If `panic_on_reenter` is `false`, this will
1337    /// return `None` if the grant region is entered and do nothing.
1338    fn access_grant<F, R>(self, fun: F, panic_on_reenter: bool) -> Option<R>
1339    where
1340        F: FnOnce(&mut GrantData<T>, &GrantKernelData) -> R,
1341    {
1342        self.access_grant_with_allocator(
1343            |grant_data, kernel_data, _allocator| fun(grant_data, kernel_data),
1344            panic_on_reenter,
1345        )
1346    }
1347
1348    /// Access the [`ProcessGrant`] memory and run a closure on the process's
1349    /// grant memory.
1350    ///
1351    /// If `panic_on_reenter` is `true`, this will panic if the grant region is
1352    /// already currently entered. If `panic_on_reenter` is `false`, this will
1353    /// return `None` if the grant region is entered and do nothing.
1354    fn access_grant_with_allocator<F, R>(self, fun: F, panic_on_reenter: bool) -> Option<R>
1355    where
1356        F: FnOnce(&mut GrantData<T>, &GrantKernelData, &mut GrantRegionAllocator) -> R,
1357    {
1358        // Access the grant that is in process memory. This can only fail if
1359        // the grant is already entered.
1360        let grant_ptr = self
1361            .process
1362            .enter_grant(self.grant_num)
1363            .map_err(|_err| {
1364                // If we get an error it is because the grant is already
1365                // entered. `process.enter_grant()` can fail for several
1366                // reasons, but only the double enter case can happen once a
1367                // grant has been applied. The other errors would be detected
1368                // earlier (i.e. before the grant can be applied).
1369
1370                // If `panic_on_reenter` is false, we skip this error and do
1371                // nothing with this grant.
1372                if !panic_on_reenter {
1373                    return;
1374                }
1375
1376                // If `enter_grant` fails, we panic!() to notify the developer
1377                // that they tried to enter the same grant twice which is
1378                // prohibited because it would result in two mutable references
1379                // existing for the same memory. This preserves type correctness
1380                // (but does crash the system).
1381                //
1382                // ## Explanation and Rationale
1383                //
1384                // This panic represents a tradeoff. While it is undesirable to
1385                // have the potential for a runtime crash in this grant region
1386                // code, it balances usability with type correctness. The
1387                // challenge is that calling `self.apps.iter()` is a common
1388                // pattern in capsules to access the grant region of every app
1389                // that is using the capsule, and sometimes it is intuitive to
1390                // call that inside of a `self.apps.enter(processid, |app| {...})`
1391                // closure. However, `.enter()` means that app's grant region is
1392                // entered, and then a naive `.iter()` would re-enter the grant
1393                // region and cause undefined behavior. We considered different
1394                // options to resolve this.
1395                //
1396                // 1. Have `.iter()` only iterate over grant regions which are
1397                //    not entered. This avoids the bug, but could lead to
1398                //    unexpected behavior, as `self.apps.iter()` will do
1399                //    different things depending on where in a capsule it is
1400                //    called.
1401                // 2. Have the compiler detect when `.iter()` is called when a
1402                //    grant region has already been entered. We don't know of a
1403                //    viable way to implement this.
1404                // 3. Panic if `.iter()` is called when a grant is already
1405                //    entered.
1406                //
1407                // We decided on option 3 because it balances minimizing
1408                // surprises (`self.apps.iter()` will always iterate all grants)
1409                // while also protecting against the bug. We expect that any
1410                // code that attempts to call `self.apps.iter()` after calling
1411                // `.enter()` will immediately encounter this `panic!()` and
1412                // have to be refactored before any tests will be successful.
1413                // Therefore, this `panic!()` should only occur at
1414                // development/testing time.
1415                //
1416                // ## How to fix this error
1417                //
1418                // If you are seeing this panic, you need to refactor your
1419                // capsule to not call `.iter()` or `.each()` from inside a
1420                // `.enter()` closure. That is, you need to close the grant
1421                // region you are currently in before trying to iterate over all
1422                // grant regions.
1423                panic!("Attempted to re-enter a grant region.");
1424            })
1425            .ok()?;
1426        let grant_t_align = GrantDataAlign(align_of::<T>());
1427        let grant_t_size = GrantDataSize(size_of::<T>());
1428
1429        let alloc_size = EnteredGrantKernelManagedLayout::grant_size(
1430            UpcallItems(Upcalls::COUNT),
1431            AllowRoItems(AllowROs::COUNT),
1432            AllowRwItems(AllowRWs::COUNT),
1433            grant_t_size,
1434            grant_t_align,
1435        );
1436
1437        // Parse layout of entire grant allocation using the known base pointer.
1438        //
1439        // # Safety
1440        //
1441        // Grant pointer is well aligned and points to initialized data.
1442        let layout = unsafe {
1443            EnteredGrantKernelManagedLayout::read_from_base(grant_ptr, self.process, self.grant_num)
1444        };
1445
1446        // Get references to all of the saved upcall data.
1447        //
1448        // # Safety
1449        //
1450        // - Pointer is well aligned and initialized with data from Self::new()
1451        //   call.
1452        // - Data will not be modified externally while this immutable reference
1453        //   is alive.
1454        // - Data is accessible for the entire duration of this immutable
1455        //   reference.
1456        // - No other mutable reference to this memory exists concurrently.
1457        //   Mutable reference to this memory are only created through the
1458        //   kernel in the syscall interface which is serialized in time with
1459        //   this call.
1460        let (saved_upcalls_slice, saved_allow_ro_slice, saved_allow_rw_slice) =
1461            layout.get_resource_slices();
1462        let grant_data = unsafe {
1463            EnteredGrantKernelManagedLayout::offset_of_grant_data_t(
1464                grant_ptr,
1465                alloc_size,
1466                grant_t_size,
1467            )
1468            .cast()
1469            .as_mut()
1470        };
1471
1472        // Create a wrapped objects that are passed to functor.
1473        let mut grant_data = GrantData::new(grant_data);
1474        let kernel_data = GrantKernelData::new(
1475            saved_upcalls_slice,
1476            saved_allow_ro_slice,
1477            saved_allow_rw_slice,
1478            self.driver_num,
1479            self.process,
1480        );
1481        // Setup an allocator in case the capsule needs additional memory in the
1482        // grant space.
1483        let mut allocator = GrantRegionAllocator {
1484            processid: self.process.processid(),
1485        };
1486
1487        // Call functor and pass back value.
1488        Some(fun(&mut grant_data, &kernel_data, &mut allocator))
1489    }
1490}
1491
1492/// Grant which was allocated from the kernel-owned grant region in a specific
1493/// process's memory, separately from a normal `Grant`.
1494///
1495/// A [`CustomGrant`] allows a capsule to allocate additional memory on behalf
1496/// of a process.
1497pub struct CustomGrant<T> {
1498    /// An identifier for this custom grant within a process's grant region.
1499    ///
1500    /// Here, this is an opaque reference that Process uses to access the
1501    /// custom grant allocation. This setup ensures that Process owns the grant
1502    /// memory.
1503    identifier: ProcessCustomGrantIdentifier,
1504
1505    /// Identifier for the process where this custom grant is allocated.
1506    processid: ProcessId,
1507
1508    /// Used to keep the Rust type of the grant.
1509    _phantom: PhantomData<T>,
1510}
1511
1512impl<T> CustomGrant<T> {
1513    /// Creates a new [`CustomGrant`].
1514    fn new(identifier: ProcessCustomGrantIdentifier, processid: ProcessId) -> Self {
1515        CustomGrant {
1516            identifier,
1517            processid,
1518            _phantom: PhantomData,
1519        }
1520    }
1521
1522    /// Helper function to get the [`ProcessId`] from the custom grant.
1523    pub fn processid(&self) -> ProcessId {
1524        self.processid
1525    }
1526
1527    /// Gives access to inner data within the given closure.
1528    ///
1529    /// If the process has since been restarted or crashed, or the memory is
1530    /// otherwise no longer present, then this function will not call the given
1531    /// closure, and will instead directly return `Err(Error::NoSuchApp)`.
1532    ///
1533    /// Because this function requires `&mut self`, it should be impossible to
1534    /// access the inner data of a given `CustomGrant` reentrantly. Thus the
1535    /// reentrance detection we use for non-custom grants is not needed here.
1536    pub fn enter<F, R>(&self, fun: F) -> Result<R, Error>
1537    where
1538        F: FnOnce(GrantData<'_, T>) -> R,
1539    {
1540        // Verify that the process this CustomGrant was allocated within still
1541        // exists.
1542        self.processid
1543            .kernel
1544            .process_map_or(Err(Error::NoSuchApp), self.processid, |process| {
1545                // App is valid.
1546
1547                // Now try to access the custom grant memory.
1548                let grant_ptr = process.enter_custom_grant(self.identifier)?;
1549
1550                // # Safety
1551                //
1552                // `grant_ptr` must be a valid pointer and there must not exist
1553                // any other references to the same memory. We verify the
1554                // pointer is valid and aligned when the memory is allocated and
1555                // `CustomGrant` is created. We are sure that there are no
1556                // other references because the only way to create a reference
1557                // is using this `enter()` function, and it can only be called
1558                // once (because of the `&mut self` requirement).
1559                let custom_grant = unsafe { &mut *(grant_ptr as *mut T) };
1560                let borrowed = GrantData::new(custom_grant);
1561                Ok(fun(borrowed))
1562            })
1563    }
1564}
1565
1566/// Tool for allocating additional memory regions in a process's grant region.
1567///
1568/// This is optionally provided along with a grant so that if a capsule needs
1569/// per-process dynamic allocation it can allocate additional memory.
1570pub struct GrantRegionAllocator {
1571    /// The process the allocator will allocate memory from.
1572    processid: ProcessId,
1573}
1574
1575impl GrantRegionAllocator {
1576    /// Allocates a new [`CustomGrant`] initialized using the given closure.
1577    ///
1578    /// The closure will be called exactly once, and the result will be used to
1579    /// initialize the owned value.
1580    ///
1581    /// This interface was chosen instead of a simple `alloc(val)` as it's
1582    /// much more likely to optimize out all stack intermediates. This
1583    /// helps to prevent stack overflows when allocating large values.
1584    ///
1585    /// # Panic Safety
1586    ///
1587    /// If `init` panics, the freshly allocated memory may leak.
1588    pub fn alloc_with<T, F>(&self, init: F) -> Result<CustomGrant<T>, Error>
1589    where
1590        F: FnOnce() -> T,
1591    {
1592        let (custom_grant_identifier, typed_ptr) = self.alloc_raw::<T>()?;
1593
1594        // # Safety
1595        //
1596        // Writing to this pointer is safe as long as the pointer is valid
1597        // and aligned. `alloc_raw()` guarantees these constraints are met.
1598        unsafe {
1599            // We use `ptr::write` to avoid `Drop`ping the uninitialized memory
1600            // in case `T` implements the `Drop` trait.
1601            write(typed_ptr.as_ptr(), init());
1602        }
1603
1604        Ok(CustomGrant::new(custom_grant_identifier, self.processid))
1605    }
1606
1607    /// Allocates a slice of n instances of a given type. Each instance is
1608    /// initialized using the provided function.
1609    ///
1610    /// The provided function will be called exactly `n` times, and will be
1611    /// passed the index it's initializing, from `0` through `NUM_ITEMS - 1`.
1612    ///
1613    /// # Panic Safety
1614    ///
1615    /// If `val_func` panics, the freshly allocated memory and any values
1616    /// already written will be leaked.
1617    pub fn alloc_n_with<T, F, const NUM_ITEMS: usize>(
1618        &self,
1619        mut init: F,
1620    ) -> Result<CustomGrant<[T; NUM_ITEMS]>, Error>
1621    where
1622        F: FnMut(usize) -> T,
1623    {
1624        let (custom_grant_identifier, typed_ptr) = self.alloc_n_raw::<T>(NUM_ITEMS)?;
1625
1626        for i in 0..NUM_ITEMS {
1627            // # Safety
1628            //
1629            // The allocate function guarantees that `ptr` points to memory
1630            // large enough to allocate `num_items` copies of the object.
1631            unsafe {
1632                write(typed_ptr.as_ptr().add(i), init(i));
1633            }
1634        }
1635
1636        Ok(CustomGrant::new(custom_grant_identifier, self.processid))
1637    }
1638
1639    /// Allocates uninitialized grant memory appropriate to store a `T`.
1640    ///
1641    /// The caller must initialize the memory.
1642    ///
1643    /// Also returns a ProcessCustomGrantIdentifier to access the memory later.
1644    fn alloc_raw<T>(&self) -> Result<(ProcessCustomGrantIdentifier, NonNull<T>), Error> {
1645        self.alloc_n_raw::<T>(1)
1646    }
1647
1648    /// Allocates space for a dynamic number of items.
1649    ///
1650    /// The caller is responsible for initializing the returned memory.
1651    ///
1652    /// Returns memory appropriate for storing `num_items` contiguous instances
1653    /// of `T` and a ProcessCustomGrantIdentifier to access the memory later.
1654    fn alloc_n_raw<T>(
1655        &self,
1656        num_items: usize,
1657    ) -> Result<(ProcessCustomGrantIdentifier, NonNull<T>), Error> {
1658        let (custom_grant_identifier, raw_ptr) =
1659            self.alloc_n_raw_inner(num_items, size_of::<T>(), align_of::<T>())?;
1660        let typed_ptr = NonNull::cast::<T>(raw_ptr);
1661
1662        Ok((custom_grant_identifier, typed_ptr))
1663    }
1664
1665    /// Helper to reduce code bloat by avoiding monomorphization.
1666    fn alloc_n_raw_inner(
1667        &self,
1668        num_items: usize,
1669        single_alloc_size: usize,
1670        alloc_align: usize,
1671    ) -> Result<(ProcessCustomGrantIdentifier, NonNull<u8>), Error> {
1672        let alloc_size = single_alloc_size
1673            .checked_mul(num_items)
1674            .ok_or(Error::OutOfMemory)?;
1675        self.processid
1676            .kernel
1677            .process_map_or(Err(Error::NoSuchApp), self.processid, |process| {
1678                process
1679                    .allocate_custom_grant(alloc_size, alloc_align)
1680                    .map_or(
1681                        Err(Error::OutOfMemory),
1682                        |(custom_grant_identifier, raw_ptr)| Ok((custom_grant_identifier, raw_ptr)),
1683                    )
1684            })
1685    }
1686}
1687
1688/// Type for storing an object of type T in process memory that is only
1689/// accessible by the kernel.
1690///
1691/// A single [`Grant`] can allocate space for one object of type T for each
1692/// process on the board. Each allocated object will reside in the grant region
1693/// belonging to the process that the object is allocated for. The [`Grant`]
1694/// type is used to get access to [`ProcessGrant`]s, which are tied to a
1695/// specific process and provide access to the memory object allocated for that
1696/// process.
1697pub struct Grant<T: Default, Upcalls: UpcallSize, AllowROs: AllowRoSize, AllowRWs: AllowRwSize> {
1698    /// Hold a reference to the core kernel so we can iterate processes.
1699    pub(crate) kernel: &'static Kernel,
1700
1701    /// Keep track of the syscall driver number assigned to the capsule that is
1702    /// using this grant. This allows us to uniquely identify upcalls stored in
1703    /// this grant.
1704    driver_num: usize,
1705
1706    /// The identifier for this grant. Having an identifier allows the Process
1707    /// implementation to lookup the memory for this grant in the specific
1708    /// process.
1709    grant_num: usize,
1710
1711    /// Used to store the Rust types for grant.
1712    ptr: PhantomData<(T, Upcalls, AllowROs, AllowRWs)>,
1713}
1714
1715impl<T: Default, Upcalls: UpcallSize, AllowROs: AllowRoSize, AllowRWs: AllowRwSize>
1716    Grant<T, Upcalls, AllowROs, AllowRWs>
1717{
1718    /// Create a new [`Grant`] type which allows a capsule to store
1719    /// process-specific data for each process in the process's memory region.
1720    ///
1721    /// This must only be called from the main kernel so that it can ensure that
1722    /// `grant_index` is a valid index.
1723    pub(crate) fn new(kernel: &'static Kernel, driver_num: usize, grant_index: usize) -> Self {
1724        Self {
1725            kernel,
1726            driver_num,
1727            grant_num: grant_index,
1728            ptr: PhantomData,
1729        }
1730    }
1731
1732    /// Enter the grant for a specific process.
1733    ///
1734    /// This creates a [`ProcessGrant`] which is a handle for a grant allocated
1735    /// for a specific process. Then, that [`ProcessGrant`] is entered and the
1736    /// provided closure is run with access to the memory in the grant region.
1737    pub fn enter<F, R>(&self, processid: ProcessId, fun: F) -> Result<R, Error>
1738    where
1739        F: FnOnce(&mut GrantData<T>, &GrantKernelData) -> R,
1740    {
1741        let pg = ProcessGrant::new(self, processid)?;
1742
1743        // If we have managed to create an `ProcessGrant`, all we need
1744        // to do is actually access the memory and run the
1745        // capsule-provided closure. This can only fail if the grant is
1746        // already entered, at which point the kernel will panic.
1747        Ok(pg.enter(fun))
1748    }
1749
1750    /// Enter the grant for a specific process with access to an allocator.
1751    ///
1752    /// This creates an [`ProcessGrant`] which is a handle for a grant allocated
1753    /// for a specific process. Then, that [`ProcessGrant`] is entered and the
1754    /// provided closure is run with access to the memory in the grant region.
1755    ///
1756    /// The allocator allows the caller to dynamically allocate additional
1757    /// memory in the process's grant region.
1758    pub fn enter_with_allocator<F, R>(&self, processid: ProcessId, fun: F) -> Result<R, Error>
1759    where
1760        F: FnOnce(&mut GrantData<T>, &GrantKernelData, &mut GrantRegionAllocator) -> R,
1761    {
1762        // Get the `ProcessGrant` for the process, possibly needing to
1763        // actually allocate the memory in the process's grant region to
1764        // do so. This can fail for a variety of reasons, and if so we
1765        // return the error to the capsule.
1766        let pg = ProcessGrant::new(self, processid)?;
1767
1768        // If we have managed to create an `ProcessGrant`, all we need
1769        // to do is actually access the memory and run the
1770        // capsule-provided closure. This can only fail if the grant is
1771        // already entered, at which point the kernel will panic.
1772        Ok(pg.enter_with_allocator(fun))
1773    }
1774
1775    /// Run a function on the grant for each active process if the grant has
1776    /// been allocated for that process.
1777    ///
1778    /// This will silently skip any process where the grant has not previously
1779    /// been allocated. This will also silently skip any invalid processes.
1780    ///
1781    /// Calling this function when an [`ProcessGrant`] for a process is
1782    /// currently entered will result in a panic.
1783    pub fn each<F>(&self, mut fun: F)
1784    where
1785        F: FnMut(ProcessId, &mut GrantData<T>, &GrantKernelData),
1786    {
1787        // Create a the iterator across `ProcessGrant`s for each process.
1788        for pg in self.iter() {
1789            let processid = pg.processid();
1790            // Since we iterating, there is no return value we need to worry
1791            // about.
1792            pg.enter(|data, upcalls| fun(processid, data, upcalls));
1793        }
1794    }
1795
1796    /// Get an iterator over all processes and their active grant regions for
1797    /// this particular grant.
1798    ///
1799    /// Calling this function when an [`ProcessGrant`] for a process is
1800    /// currently entered will result in a panic.
1801    pub fn iter(&self) -> Iter<T, Upcalls, AllowROs, AllowRWs> {
1802        Iter {
1803            grant: self,
1804            subiter: self.kernel.get_process_iter(),
1805        }
1806    }
1807}
1808
1809/// Type to iterate [`ProcessGrant`]s across processes.
1810pub struct Iter<
1811    'a,
1812    T: 'a + Default,
1813    Upcalls: UpcallSize,
1814    AllowROs: AllowRoSize,
1815    AllowRWs: AllowRwSize,
1816> {
1817    /// The grant type to use.
1818    grant: &'a Grant<T, Upcalls, AllowROs, AllowRWs>,
1819
1820    /// Iterator over valid processes.
1821    subiter: core::iter::FilterMap<
1822        core::slice::Iter<'a, ProcessSlot>,
1823        fn(&ProcessSlot) -> Option<&'static dyn Process>,
1824    >,
1825}
1826
1827impl<'a, T: Default, Upcalls: UpcallSize, AllowROs: AllowRoSize, AllowRWs: AllowRwSize> Iterator
1828    for Iter<'a, T, Upcalls, AllowROs, AllowRWs>
1829{
1830    type Item = ProcessGrant<'a, T, Upcalls, AllowROs, AllowRWs>;
1831
1832    fn next(&mut self) -> Option<Self::Item> {
1833        let grant = self.grant;
1834        // Get the next `ProcessId` from the kernel processes array that is
1835        // setup to use this grant. Since the iterator itself is saved calling
1836        // this function again will start where we left off.
1837        self.subiter
1838            .find_map(|process| ProcessGrant::new_if_allocated(grant, process))
1839    }
1840}