kernel/
grant.rs

1// Licensed under the Apache License, Version 2.0 or the MIT License.
2// SPDX-License-Identifier: Apache-2.0 OR MIT
3// Copyright Tock Contributors 2022.
4
5//! Support for processes granting memory from their allocations to the kernel.
6//!
7//! ## Grant Overview
8//!
9//! Grants allow capsules to dynamically allocate memory from a process to hold
10//! state on the process's behalf.
11//!
12//! Each capsule that wishes to do this needs to have a [`Grant`] type. Grants
13//! are created at boot, and each have a unique ID and a type `T`. This type
14//! only allows the capsule to allocate memory from a process in the future. It
15//! does not initially represent any allocated memory.
16//!
17//! When a capsule does wish to use its Grant to allocate memory from a process,
18//! it must "enter" the Grant with a specific [`ProcessId`]. Entering a Grant
19//! for a specific process instructs the core kernel to create an object `T` in
20//! the process's memory space and provide the capsule with access to it. If the
21//! Grant has not previously been entered for that process, the memory for
22//! object `T` will be allocated from the "grant region" within the
23//! kernel-accessible portion of the process's memory.
24//!
25//! If a Grant has never been entered for a process, the object `T` will _not_
26//! be allocated in that process's grant region, even if the `Grant` has been
27//! entered for other processes.
28//!
29//! Upcalls and allowed buffer references are stored in the dynamically
30//! allocated grant for a particular Driver as well. Upcalls and allowed buffer
31//! references are stored outside of the `T` object to enable the kernel to
32//! manage them and ensure swapping guarantees are met.
33//!
34//! The type `T` of a Grant is fixed in size and the number of upcalls and
35//! allowed buffers associated with a grant is fixed. That is, when a Grant is
36//! entered for a process the resulting allocated object will be the size of
37//! `SizeOf<T>` plus the size for the structure to hold upcalls and allowed
38//! buffer references. If capsules need additional process-specific memory for
39//! their operation, they can use an [`GrantRegionAllocator`] to request
40//! additional memory from the process's grant region.
41//!
42//! ```text,ignore
43//!                            ┌──────────────────┐
44//!                            │                  │
45//!                            │ Capsule          │
46//!                            │                  │
47//!                            └─┬────────────────┘
48//!                              │ Capsules hold
49//!                              │ references to
50//!                              │ grants.
51//!                              ▼
52//!                            ┌──────────────────┐
53//!                            │ Grant            │
54//!                            │                  │
55//!  Process Memory            │ Type: T          │
56//! ┌────────────────────────┐ │ grant_num: 1     │
57//! │                        │ │ driver_num: 0x4  │
58//! │  ...                   │ └───┬─────────────┬┘
59//! ├────────────────────────┤     │Each Grant   │
60//! │ Grant       ptr 0      │     │has a pointer│
61//! │ Pointers    ptr 1 ───┐ │ ◄───┘per process. │
62//! │             ...      │ │                   │
63//! │             ptr N    │ │                   │
64//! ├──────────────────────┼─┤                   │
65//! │  ...                 │ │                   │
66//! ├──────────────────────┼─┤                   │
67//! │ Grant Region         │ │     When a Grant  │
68//! │                      │ │     is allocated  │
69//! │ ┌─────────────────┐  │ │     for a process │
70//! │ │ Allocated Grant │  │ │ ◄─────────────────┘
71//! │ │                 │  │ │     it uses memory
72//! │ │  [ SizeOf<T> ]  │  │ │     from the grant
73//! │ │─────────────────│  │ │     region.
74//! │ │ Padding         │  │ │
75//! │ │─────────────────│  │ │
76//! │ │ GrantKernelData │  │ │
77//! │ └─────────────────┘◄─┘ │
78//! │                        │
79//! │ ┌─────────────────┐    │
80//! │ │ Custom Grant    │    │ ◄── Capsules can
81//! │ │                 │    │     allocate extra
82//! │ └─────────────────┘    │     memory if needed.
83//! │                        │
84//! ├─kernel_brk─────────────┤
85//! │                        │
86//! │ ...                    │
87//! └────────────────────────┘
88//! ```
89//!
90//! ## Grant Mechanisms and Types
91//!
92//! Here is an overview of the types used by grant.rs to implement the Grant
93//! functionality in Tock:
94//!
95//! ```text,ignore
96//!                         ┌──────────────────────────┐
97//!                         │ struct Grant<T, ...> {   │
98//!                         │   driver_num: usize      │
99//!                         │   grant_num: usize       │
100//!                         │ }                        ├───┐
101//! Entering a Grant for a  └──┬───────────────────────┘   │
102//! process causes the         │                           │
103//! memory for T to be         │ .enter(ProcessId)         │ .enter(ProcessId, fn)
104//! allocated.                 ▼                           │
105//!                         ┌──────────────────────────┐   │ For convenience,
106//! ProcessGrant represents │ struct ProcessGrant<T> { │   │ allocating and getting
107//! a Grant allocated for a │   number: usize          │   │ access to the T object
108//! specific process.       │   process: &Process      │   │ is combined in one
109//!                         │ }                        │   │ .enter() call.
110//! A provided closure      └──┬───────────────────────┘   │
111//! is given access to         │                           │
112//! the underlying memory      │ .enter(fn)                │
113//! where the T is stored.     ▼                           │
114//!                         ┌────────────────────────────┐ │
115//! GrantData wraps the     │ struct GrantData<T>   {    │◄┘
116//! type and provides       │   data: &mut T             │
117//! mutable access.         │ }                          │
118//! GrantKernelData         │ struct GrantKernelData {   │
119//! provides access to      │   upcalls: [SavedUpcall]   │
120//! scheduling upcalls      │   allow_ro: [SavedAllowRo] │
121//! and process buffers.    │   allow_rw: [SavedAllowRW] │
122//!                         │ }                          │
123//!                         └──┬─────────────────────────┘
124//! The actual object T can    │
125//! only be accessed inside    │ fn(mem: &GrantData, kernel_data: &GrantKernelData)
126//! the closure.               ▼
127//! ```
128
129use core::cmp;
130use core::marker::PhantomData;
131use core::mem::{align_of, size_of};
132use core::ops::{Deref, DerefMut};
133use core::ptr::{write, NonNull};
134use core::slice;
135
136use crate::kernel::Kernel;
137use crate::process::{Error, Process, ProcessCustomGrantIdentifier, ProcessId};
138use crate::processbuffer::{ReadOnlyProcessBuffer, ReadWriteProcessBuffer};
139use crate::processbuffer::{ReadOnlyProcessBufferRef, ReadWriteProcessBufferRef};
140use crate::upcall::{Upcall, UpcallError, UpcallId};
141use crate::utilities::capability_ptr::CapabilityPtr;
142use crate::utilities::machine_register::MachineRegister;
143use crate::ErrorCode;
144
145/// Tracks how many upcalls a grant instance supports automatically.
146pub trait UpcallSize {
147    /// The number of upcalls the grant supports.
148    const COUNT: u8;
149}
150
151/// Specifies how many upcalls a grant instance supports automatically.
152pub struct UpcallCount<const NUM: u8>;
153impl<const NUM: u8> UpcallSize for UpcallCount<NUM> {
154    const COUNT: u8 = NUM;
155}
156
157/// Tracks how many read-only allows a grant instance supports automatically.
158pub trait AllowRoSize {
159    /// The number of read-only allows the grant supports.
160    const COUNT: u8;
161}
162
163/// Specifies how many read-only allows a grant instance supports automatically.
164pub struct AllowRoCount<const NUM: u8>;
165impl<const NUM: u8> AllowRoSize for AllowRoCount<NUM> {
166    const COUNT: u8 = NUM;
167}
168
169/// Tracks how many read-write allows a grant instance supports automatically.
170pub trait AllowRwSize {
171    /// The number of read-write allows the grant supports.
172    const COUNT: u8;
173}
174
175/// Specifies how many read-write allows a grant instance supports
176/// automatically.
177pub struct AllowRwCount<const NUM: u8>;
178impl<const NUM: u8> AllowRwSize for AllowRwCount<NUM> {
179    const COUNT: u8 = NUM;
180}
181
182/// Helper that calculated offsets within the kernel owned memory (i.e. the
183/// non-T part of grant).
184///
185/// Example layout of full grant belonging to a single app and driver:
186///
187/// ```text,ignore
188/// 0x003FFC8  ┌────────────────────────────────────┐
189///            │   T                                |
190/// 0x003FFxx  ├  ───────────────────────── ┐ K     |
191///            │   Padding (ensure T aligns)| e     |
192/// 0x003FF44  ├  ───────────────────────── | r     |
193///            │   SavedAllowRwN            | n     |
194///            │   ...                      | e     | G
195///            │   SavedAllowRw1            | l     | r
196///            │   SavedAllowRw0            |       | a
197/// 0x003FF44  ├  ───────────────────────── | O     | n
198///            │   SavedAllowRoN            | w     | t
199///            │   ...                      | n     |
200///            │   SavedAllowRo1            | e     | M
201///            │   SavedAllowRo0            | d     | e
202/// 0x003FF30  ├  ───────────────────────── |       | m
203///            │   SavedUpcallN             | D     | o
204///            │   ...                      | a     | r
205///            │   SavedUpcall1             | t     | y
206///            │   SavedUpcall0             | a     |
207/// 0x003FF24  ├  ───────────────────────── |       |
208///            │   Counters (usize)         |       |
209/// 0x003FF20  └────────────────────────────────────┘
210/// ```
211///
212/// The counters structure is composed as:
213///
214/// ```text,ignore
215/// 0             1             2             3         bytes
216/// |-------------|-------------|-------------|-------------|
217/// | # Upcalls   | # RO Allows | # RW Allows | [unused]    |
218/// |-------------|-------------|-------------|-------------|
219/// ```
220///
221/// This type is created whenever a grant is entered, and is responsible for
222/// ensuring that the grant is closed when it is no longer used. On `Drop`, we
223/// leave the grant. This protects against calling `grant.enter()` without
224/// calling the corresponding `grant.leave()`, perhaps due to accidentally using
225/// the `?` operator.
226struct EnteredGrantKernelManagedLayout<'a> {
227    /// Leaving a grant is handled through the process implementation, so must
228    /// keep a reference to the relevant process.
229    process: &'a dyn Process,
230    /// The grant number of the entered grant that we want to ensure we leave
231    /// properly.
232    grant_num: usize,
233
234    /// The location of the counters structure for the grant.
235    counters_ptr: *mut usize,
236    /// Pointer to the array of saved upcalls.
237    upcalls_array: *mut SavedUpcall,
238    /// Pointer to the array of saved read-only allows.
239    allow_ro_array: *mut SavedAllowRo,
240    /// Pointer to the array of saved read-write allows.
241    allow_rw_array: *mut SavedAllowRw,
242}
243
244/// Represents the number of the upcall elements in the kernel owned section of
245/// the grant.
246#[derive(Copy, Clone)]
247struct UpcallItems(u8);
248/// Represents the number of the read-only allow elements in the kernel owned
249/// section of the grant.
250#[derive(Copy, Clone)]
251struct AllowRoItems(u8);
252/// Represents the number of the read-write allow elements in the kernel owned
253/// section of the grant.
254#[derive(Copy, Clone)]
255struct AllowRwItems(u8);
256/// Represents the size data (in bytes) T within the grant.
257#[derive(Copy, Clone)]
258struct GrantDataSize(usize);
259/// Represents the alignment of data T within the grant.
260#[derive(Copy, Clone)]
261struct GrantDataAlign(usize);
262
263impl<'a> EnteredGrantKernelManagedLayout<'a> {
264    /// Reads the specified pointer as the base of the kernel owned grant region
265    /// that has previously been initialized.
266    ///
267    /// # Safety
268    ///
269    /// The incoming base pointer must be well aligned and already contain
270    /// initialized data in the expected form. There must not be any other
271    /// `EnteredGrantKernelManagedLayout` for the given `base_ptr` at the same
272    /// time, otherwise multiple mutable references to the same upcall/allow
273    /// slices could be created.
274    unsafe fn read_from_base(
275        base_ptr: NonNull<u8>,
276        process: &'a dyn Process,
277        grant_num: usize,
278    ) -> Self {
279        let counters_ptr = base_ptr.as_ptr() as *mut usize;
280        let counters_val = counters_ptr.read();
281
282        // Parse the counters field for each of the fields
283        let [_, _, allow_ro_num, upcalls_num] = u32::to_be_bytes(counters_val as u32);
284
285        // Skip over the counter usize, then the stored array of `SavedAllowRo`
286        // items and `SavedAllowRw` items.
287        let upcalls_array = counters_ptr.add(1) as *mut SavedUpcall;
288        let allow_ro_array = upcalls_array.add(upcalls_num as usize) as *mut SavedAllowRo;
289        let allow_rw_array = allow_ro_array.add(allow_ro_num as usize) as *mut SavedAllowRw;
290
291        Self {
292            process,
293            grant_num,
294            counters_ptr,
295            upcalls_array,
296            allow_ro_array,
297            allow_rw_array,
298        }
299    }
300
301    /// Creates a layout from the specified pointer and lengths of arrays and
302    /// initializes the kernel owned portion of the layout.
303    ///
304    /// # Safety
305    ///
306    /// The incoming base pointer must be well aligned and reference enough
307    /// memory to hold the entire kernel managed grant structure. There must
308    /// not be any other `EnteredGrantKernelManagedLayout` for
309    /// the given `base_ptr` at the same time, otherwise multiple mutable
310    /// references to the same upcall/allow slices could be created.
311    unsafe fn initialize_from_counts(
312        base_ptr: NonNull<u8>,
313        upcalls_num_val: UpcallItems,
314        allow_ro_num_val: AllowRoItems,
315        allow_rw_num_val: AllowRwItems,
316        process: &'a dyn Process,
317        grant_num: usize,
318    ) -> Self {
319        let counters_ptr = base_ptr.as_ptr() as *mut usize;
320
321        // Create the counters usize value by correctly packing the various
322        // counts into 8 bit fields.
323        let counter: usize =
324            u32::from_be_bytes([0, allow_rw_num_val.0, allow_ro_num_val.0, upcalls_num_val.0])
325                as usize;
326
327        let upcalls_array = counters_ptr.add(1) as *mut SavedUpcall;
328        let allow_ro_array = upcalls_array.add(upcalls_num_val.0.into()) as *mut SavedAllowRo;
329        let allow_rw_array = allow_ro_array.add(allow_ro_num_val.0.into()) as *mut SavedAllowRw;
330
331        counters_ptr.write(counter);
332        write_default_array(upcalls_array, upcalls_num_val.0.into());
333        write_default_array(allow_ro_array, allow_ro_num_val.0.into());
334        write_default_array(allow_rw_array, allow_rw_num_val.0.into());
335
336        Self {
337            process,
338            grant_num,
339            counters_ptr,
340            upcalls_array,
341            allow_ro_array,
342            allow_rw_array,
343        }
344    }
345
346    /// Returns the entire grant size including the kernel owned memory,
347    /// padding, and data for T. Requires that grant_t_align be a power of 2,
348    /// which is guaranteed from align_of rust calls.
349    fn grant_size(
350        upcalls_num: UpcallItems,
351        allow_ro_num: AllowRoItems,
352        allow_rw_num: AllowRwItems,
353        grant_t_size: GrantDataSize,
354        grant_t_align: GrantDataAlign,
355    ) -> usize {
356        let kernel_managed_size = size_of::<usize>()
357            + upcalls_num.0 as usize * size_of::<SavedUpcall>()
358            + allow_ro_num.0 as usize * size_of::<SavedAllowRo>()
359            + allow_rw_num.0 as usize * size_of::<SavedAllowRw>();
360        // We know that grant_t_align is a power of 2, so we can make a mask
361        // that will save only the remainder bits.
362        let grant_t_align_mask = grant_t_align.0 - 1;
363        // Determine padding to get to the next multiple of grant_t_align by
364        // taking the remainder and subtracting that from the alignment, then
365        // ensuring a full alignment value maps to 0.
366        let padding =
367            (grant_t_align.0 - (kernel_managed_size & grant_t_align_mask)) & grant_t_align_mask;
368        kernel_managed_size + padding + grant_t_size.0
369    }
370
371    /// Returns the alignment of the entire grant region based on the alignment
372    /// of data T.
373    fn grant_align(grant_t_align: GrantDataAlign) -> usize {
374        // The kernel owned memory all aligned to usize. We need to use the
375        // higher of the two alignment to ensure our padding calculations work
376        // for any alignment of T.
377        cmp::max(align_of::<usize>(), grant_t_align.0)
378    }
379
380    /// Returns the offset for the grant data t within the entire grant region.
381    ///
382    /// # Safety
383    ///
384    /// The caller must ensure that the specified base pointer is aligned to at
385    /// least the alignment of T and points to a grant that is of size
386    /// grant_size bytes.
387    unsafe fn offset_of_grant_data_t(
388        base_ptr: NonNull<u8>,
389        grant_size: usize,
390        grant_t_size: GrantDataSize,
391    ) -> NonNull<u8> {
392        // The location of the grant data T is the last element in the entire
393        // grant region. Caller must verify that memory is accessible and well
394        // aligned to T.
395        let grant_t_size_usize: usize = grant_t_size.0;
396        NonNull::new_unchecked(base_ptr.as_ptr().add(grant_size - grant_t_size_usize))
397    }
398
399    /// Read an 8 bit value from the counter field offset by the specified
400    /// number of bits. This is a helper function for reading the counter field.
401    fn get_counter_offset(&self, offset_bits: usize) -> usize {
402        // # Safety
403        //
404        // Creating a `EnteredGrantKernelManagedLayout` object requires that the
405        // pointers are well aligned and point to valid memory.
406        let counters_val = unsafe { self.counters_ptr.read() };
407        (counters_val >> offset_bits) & 0xFF
408    }
409
410    /// Return the number of upcalls stored by the core kernel for this grant.
411    fn get_upcalls_number(&self) -> usize {
412        self.get_counter_offset(0)
413    }
414
415    /// Return the number of read-only allow buffers stored by the core kernel
416    /// for this grant.
417    fn get_allow_ro_number(&self) -> usize {
418        self.get_counter_offset(8)
419    }
420
421    /// Return the number of read-write allow buffers stored by the core kernel
422    /// for this grant.
423    fn get_allow_rw_number(&self) -> usize {
424        self.get_counter_offset(16)
425    }
426
427    /// Return mutable access to the slice of stored upcalls for this grant.
428    /// This is necessary for storing a new upcall.
429    fn get_upcalls_slice(&mut self) -> &mut [SavedUpcall] {
430        // # Safety
431        //
432        // Creating a `EnteredGrantKernelManagedLayout` object ensures that the
433        // pointer to the upcall array is valid.
434        unsafe { slice::from_raw_parts_mut(self.upcalls_array, self.get_upcalls_number()) }
435    }
436
437    /// Return mutable access to the slice of stored read-only allow buffers for
438    /// this grant. This is necessary for storing a new read-only allow.
439    fn get_allow_ro_slice(&mut self) -> &mut [SavedAllowRo] {
440        // # Safety
441        //
442        // Creating a `EnteredGrantKernelManagedLayout` object ensures that the
443        // pointer to the RO allow array is valid.
444        unsafe { slice::from_raw_parts_mut(self.allow_ro_array, self.get_allow_ro_number()) }
445    }
446
447    /// Return mutable access to the slice of stored read-write allow buffers
448    /// for this grant. This is necessary for storing a new read-write allow.
449    fn get_allow_rw_slice(&mut self) -> &mut [SavedAllowRw] {
450        // # Safety
451        //
452        // Creating a `EnteredGrantKernelManagedLayout` object ensures that the
453        // pointer to the RW allow array is valid.
454        unsafe { slice::from_raw_parts_mut(self.allow_rw_array, self.get_allow_rw_number()) }
455    }
456
457    /// Return slices to the kernel managed upcalls and allow buffers. This
458    /// permits using upcalls and allow buffers when a capsule is accessing a
459    /// grant.
460    fn get_resource_slices(&self) -> (&[SavedUpcall], &[SavedAllowRo], &[SavedAllowRw]) {
461        // # Safety
462        //
463        // Creating a `EnteredGrantKernelManagedLayout` object ensures that the
464        // pointer to the upcall array is valid.
465        let upcall_slice =
466            unsafe { slice::from_raw_parts(self.upcalls_array, self.get_upcalls_number()) };
467
468        // # Safety
469        //
470        // Creating a `EnteredGrantKernelManagedLayout` object ensures that the
471        // pointer to the RO allow array is valid.
472        let allow_ro_slice =
473            unsafe { slice::from_raw_parts(self.allow_ro_array, self.get_allow_ro_number()) };
474
475        // # Safety
476        //
477        // Creating a `KernelManagedLayout` object ensures that the pointer to
478        // the RW allow array is valid.
479        let allow_rw_slice =
480            unsafe { slice::from_raw_parts(self.allow_rw_array, self.get_allow_rw_number()) };
481
482        (upcall_slice, allow_ro_slice, allow_rw_slice)
483    }
484}
485
486// Ensure that we leave the grant once this goes out of scope.
487impl Drop for EnteredGrantKernelManagedLayout<'_> {
488    fn drop(&mut self) {
489        // ### Safety
490        //
491        // To safely call this function we must ensure that no references will
492        // exist to the grant once `leave_grant()` returns. Because using a
493        // `EnteredGrantKernelManagedLayout` object is the only only way we
494        // access the actual memory of a grant, and we are calling
495        // `leave_grant()` from its `drop()` method, we are sure there will be
496        // no remaining references to the grant.
497        unsafe {
498            self.process.leave_grant(self.grant_num);
499        }
500    }
501}
502
503/// This [`GrantData`] object provides access to the memory allocated for a
504/// grant for a specific process.
505///
506/// The [`GrantData`] type is templated on `T`, the actual type of the object in
507/// the grant. [`GrantData'] holds a mutable reference to the type, allowing
508/// users access to the object in process memory.
509///
510/// Capsules gain access to a [`GrantData`] object by calling
511/// [`Grant::enter()`].
512pub struct GrantData<'a, T: 'a + ?Sized> {
513    /// The mutable reference to the actual object type stored in the grant.
514    data: &'a mut T,
515}
516
517impl<'a, T: 'a + ?Sized> GrantData<'a, T> {
518    /// Create a [`GrantData`] object to provide access to the actual object
519    /// allocated for a process.
520    ///
521    /// Only one can [`GrantData`] per underlying object can be created at a
522    /// time. Otherwise, there would be multiple mutable references to the same
523    /// object which is undefined behavior.
524    fn new(data: &'a mut T) -> GrantData<'a, T> {
525        GrantData { data }
526    }
527}
528
529impl<'a, T: 'a + ?Sized> Deref for GrantData<'a, T> {
530    type Target = T;
531    fn deref(&self) -> &T {
532        self.data
533    }
534}
535
536impl<'a, T: 'a + ?Sized> DerefMut for GrantData<'a, T> {
537    fn deref_mut(&mut self) -> &mut T {
538        self.data
539    }
540}
541
542/// This [`GrantKernelData`] object provides a handle to access upcalls and
543/// process buffers stored on behalf of a particular grant/driver.
544///
545/// Capsules gain access to a [`GrantKernelData`] object by calling
546/// [`Grant::enter()`]. From there, they can schedule upcalls or access process
547/// buffers.
548///
549/// It is expected that this type will only exist as a short-lived stack
550/// allocation, so its size is not a significant concern.
551pub struct GrantKernelData<'a> {
552    /// A reference to the actual upcall slice stored in the grant.
553    upcalls: &'a [SavedUpcall],
554
555    /// A reference to the actual read only allow slice stored in the grant.
556    allow_ro: &'a [SavedAllowRo],
557
558    /// A reference to the actual read write allow slice stored in the grant.
559    allow_rw: &'a [SavedAllowRw],
560
561    /// We need to keep track of the driver number so we can properly identify
562    /// the Upcall that is called. We need to keep track of its source so we can
563    /// remove it if the Upcall is unsubscribed.
564    driver_num: usize,
565
566    /// A reference to the process that these upcalls are for. This is used for
567    /// actually scheduling the upcalls.
568    process: &'a dyn Process,
569}
570
571impl<'a> GrantKernelData<'a> {
572    /// Create a [`GrantKernelData`] object to provide a handle for capsules to
573    /// call Upcalls.
574    fn new(
575        upcalls: &'a [SavedUpcall],
576        allow_ro: &'a [SavedAllowRo],
577        allow_rw: &'a [SavedAllowRw],
578        driver_num: usize,
579        process: &'a dyn Process,
580    ) -> GrantKernelData<'a> {
581        Self {
582            upcalls,
583            allow_ro,
584            allow_rw,
585            driver_num,
586            process,
587        }
588    }
589
590    /// Schedule the specified upcall for the process with r0, r1, r2 as
591    /// provided values.
592    ///
593    /// Capsules call this function to schedule upcalls, and upcalls are
594    /// identified by the `subscribe_num`, which must match the subscribe number
595    /// used when the upcall was originally subscribed by a process.
596    /// `subscribe_num`s are indexed starting at zero.
597    pub fn schedule_upcall(
598        &self,
599        subscribe_num: usize,
600        r: (usize, usize, usize),
601    ) -> Result<(), UpcallError> {
602        // Implement `self.upcalls[subscribe_num]` without a chance of a panic.
603        self.upcalls.get(subscribe_num).map_or(
604            Err(UpcallError::InvalidSubscribeNum),
605            |saved_upcall| {
606                // We can create an `Upcall` object based on what is stored in
607                // the process grant and use that to add the upcall to the
608                // pending array for the process.
609                let upcall = Upcall::new(
610                    self.process.processid(),
611                    UpcallId {
612                        subscribe_num,
613                        driver_num: self.driver_num,
614                    },
615                    saved_upcall.appdata,
616                    saved_upcall.fn_ptr,
617                );
618                upcall.schedule(self.process, r.0, r.1, r.2)
619            },
620        )
621    }
622
623    /// Search the work queue for the first pending operation with the given
624    /// `subscribe_num` and if one exists remove it from the task queue.
625    ///
626    /// Returns the associated [`Task`](crate::process::Task) if one was found, otherwise returns
627    /// [`None`].
628    pub fn remove_upcall(&self, subscribe_num: usize) -> Option<crate::process::Task> {
629        self.process.remove_upcall(UpcallId {
630            subscribe_num,
631            driver_num: self.driver_num,
632        })
633    }
634
635    /// Remove all scheduled upcalls with the given `subscribe_num` from the
636    /// task queue.
637    ///
638    /// Returns the number of removed upcalls.
639    pub fn remove_pending_upcalls(&self, subscribe_num: usize) -> usize {
640        self.process.remove_pending_upcalls(UpcallId {
641            subscribe_num,
642            driver_num: self.driver_num,
643        })
644    }
645
646    /// Returns a lifetime limited reference to the requested
647    /// [`ReadOnlyProcessBuffer`].
648    ///
649    /// The len of the returned [`ReadOnlyProcessBuffer`] must be checked by the
650    /// caller to ensure that a buffer has in fact been allocated. An
651    /// unallocated buffer will be returned as a [`ReadOnlyProcessBuffer`] of
652    /// length 0.
653    ///
654    /// The [`ReadOnlyProcessBuffer`] is only valid for as long as this object
655    /// is valid, i.e. the lifetime of the app enter closure.
656    ///
657    /// If the specified allow number is invalid, then a
658    /// [`crate::process::Error::AddressOutOfBounds`] will be returned. This
659    /// returns a [`crate::process::Error`] to allow for easy chaining of this
660    /// function with the `ReadOnlyProcessBuffer::enter()` function with
661    /// `and_then`.
662    pub fn get_readonly_processbuffer(
663        &self,
664        allow_ro_num: usize,
665    ) -> Result<ReadOnlyProcessBufferRef, crate::process::Error> {
666        self.allow_ro.get(allow_ro_num).map_or(
667            Err(crate::process::Error::AddressOutOfBounds),
668            |saved_ro| {
669                // # Safety
670                //
671                // This is the saved process buffer data has been validated to
672                // be wholly contained within this process before it was stored.
673                // The lifetime of the ReadOnlyProcessBuffer is bound to the
674                // lifetime of self, which correctly limits dereferencing this
675                // saved pointer to only when it is valid.
676                unsafe {
677                    Ok(ReadOnlyProcessBufferRef::new(
678                        saved_ro.ptr,
679                        saved_ro.len,
680                        self.process.processid(),
681                    ))
682                }
683            },
684        )
685    }
686
687    /// Returns a lifetime limited reference to the requested
688    /// [`ReadWriteProcessBuffer`].
689    ///
690    /// The length of the returned [`ReadWriteProcessBuffer`] must be checked by
691    /// the caller to ensure that a buffer has in fact been allocated. An
692    /// unallocated buffer will be returned as a [`ReadWriteProcessBuffer`] of
693    /// length 0.
694    ///
695    /// The [`ReadWriteProcessBuffer`] is only value for as long as this object
696    /// is valid, i.e. the lifetime of the app enter closure.
697    ///
698    /// If the specified allow number is invalid, then a
699    /// [`crate::process::Error::AddressOutOfBounds`] will be returned. This
700    /// returns a [`crate::process::Error`] to allow for easy chaining of this
701    /// function with the `ReadWriteProcessBuffer::enter()` function with
702    /// `and_then`.
703    pub fn get_readwrite_processbuffer(
704        &self,
705        allow_rw_num: usize,
706    ) -> Result<ReadWriteProcessBufferRef, crate::process::Error> {
707        self.allow_rw.get(allow_rw_num).map_or(
708            Err(crate::process::Error::AddressOutOfBounds),
709            |saved_rw| {
710                // # Safety
711                //
712                // This is the saved process buffer data has been validated to
713                // be wholly contained within this process before it was stored.
714                // The lifetime of the ReadWriteProcessBuffer is bound to the
715                // lifetime of self, which correctly limits dereferencing this
716                // saved pointer to only when it is valid.
717                unsafe {
718                    Ok(ReadWriteProcessBufferRef::new(
719                        saved_rw.ptr,
720                        saved_rw.len,
721                        self.process.processid(),
722                    ))
723                }
724            },
725        )
726    }
727}
728
729/// A minimal representation of an upcall, used for storing an upcall in a
730/// process' grant table without wasting memory duplicating information such as
731/// process ID.
732#[repr(C)]
733#[derive(Default)]
734struct SavedUpcall {
735    appdata: MachineRegister,
736    fn_ptr: CapabilityPtr,
737}
738
739/// A minimal representation of a read-only allow from app, used for storing a
740/// read-only allow in a process' kernel managed grant space without wasting
741/// memory duplicating information such as process ID.
742#[repr(C)]
743struct SavedAllowRo {
744    ptr: *const u8,
745    len: usize,
746}
747
748// This allow is still needed on the current stable compiler, but generates a warning
749// on the current nightly compiler, as of 05/18/2025. So allow this warning for now.
750// This can probably be fixed on the next nightly update.
751#[allow(clippy::derivable_impls)]
752impl Default for SavedAllowRo {
753    fn default() -> Self {
754        Self {
755            ptr: core::ptr::null(),
756            len: 0,
757        }
758    }
759}
760
761/// A minimal representation of a read-write allow from app, used for storing a
762/// read-write allow in a process' kernel managed grant space without wasting
763/// memory duplicating information such as process ID.
764#[repr(C)]
765struct SavedAllowRw {
766    ptr: *mut u8,
767    len: usize,
768}
769
770// This allow is still needed on the current stable compiler, but generates a warning
771// on the current nightly compiler, as of 05/18/2025. So allow this warning for now.
772// This can probably be fixed on the next nightly update.
773#[allow(clippy::derivable_impls)]
774impl Default for SavedAllowRw {
775    fn default() -> Self {
776        Self {
777            ptr: core::ptr::null_mut(),
778            len: 0,
779        }
780    }
781}
782
783/// Write the default value of T to every element of the array.
784///
785/// # Safety
786///
787/// The pointer must be well aligned and point to allocated memory that is
788/// writable for `size_of::<T> * num` bytes. No Rust references may exist to
789/// memory in the address range spanned by `base..base+num` at the time this
790/// function is called. The memory does not need to be initialized yet. If it
791/// already does contain initialized memory, then those contents will be
792/// overwritten without being `Drop`ed first.
793unsafe fn write_default_array<T: Default>(base: *mut T, num: usize) {
794    for i in 0..num {
795        base.add(i).write(T::default());
796    }
797}
798
799/// Enters the grant for the specified process. Caller must hold on to the grant
800/// lifetime guard while they accessing the memory in the layout (second
801/// element).
802fn enter_grant_kernel_managed(
803    process: &dyn Process,
804    driver_num: usize,
805) -> Result<EnteredGrantKernelManagedLayout, ErrorCode> {
806    let grant_num = process.lookup_grant_from_driver_num(driver_num)?;
807
808    // Check if the grant has been allocated, and if not we cannot enter this
809    // grant.
810    match process.grant_is_allocated(grant_num) {
811        Some(true) => { /* Allocated, nothing to do */ }
812        Some(false) => return Err(ErrorCode::NOMEM),
813        None => return Err(ErrorCode::FAIL),
814    }
815
816    // Return early if no grant.
817    let grant_base_ptr = process.enter_grant(grant_num).or(Err(ErrorCode::NOMEM))?;
818    // # Safety
819    //
820    // We know that this pointer is well aligned and initialized with meaningful
821    // data when the grant region was allocated.
822    let layout = unsafe {
823        EnteredGrantKernelManagedLayout::read_from_base(grant_base_ptr, process, grant_num)
824    };
825    Ok(layout)
826}
827
828/// Subscribe to an upcall by saving the upcall in the grant region for the
829/// process and returning the existing upcall for the same UpcallId.
830pub(crate) fn subscribe(
831    process: &dyn Process,
832    upcall: Upcall,
833) -> Result<Upcall, (Upcall, ErrorCode)> {
834    // Enter grant and keep it open until _grant_open goes out of scope.
835    let mut layout = match enter_grant_kernel_managed(process, upcall.upcall_id.driver_num) {
836        Ok(val) => val,
837        Err(e) => return Err((upcall, e)),
838    };
839
840    // Create the saved upcalls slice from the grant memory.
841    //
842    // # Safety
843    //
844    // This is safe because of how the grant was initially allocated and that
845    // because we were able to enter the grant the grant region must be valid
846    // and initialized. We are also holding the grant open until `_grant_open`
847    // goes out of scope.
848    let saved_upcalls_slice = layout.get_upcalls_slice();
849
850    // Index into the saved upcall slice to get the old upcall. Use .get in case
851    // userspace passed us a bad subscribe number.
852    match saved_upcalls_slice.get_mut(upcall.upcall_id.subscribe_num) {
853        Some(saved_upcall) => {
854            // Create an `Upcall` object with the old saved upcall.
855            let old_upcall = Upcall::new(
856                process.processid(),
857                upcall.upcall_id,
858                saved_upcall.appdata,
859                saved_upcall.fn_ptr,
860            );
861
862            // Overwrite the saved upcall with the new upcall.
863            saved_upcall.appdata = upcall.appdata;
864            saved_upcall.fn_ptr = upcall.fn_ptr;
865
866            // Success!
867            Ok(old_upcall)
868        }
869        None => Err((upcall, ErrorCode::NOSUPPORT)),
870    }
871}
872
873/// Stores the specified read-only process buffer in the kernel managed grant
874/// region for this process and driver. The previous read-only process buffer
875/// stored at the same allow_num id is returned.
876pub(crate) fn allow_ro(
877    process: &dyn Process,
878    driver_num: usize,
879    allow_num: usize,
880    buffer: ReadOnlyProcessBuffer,
881) -> Result<ReadOnlyProcessBuffer, (ReadOnlyProcessBuffer, ErrorCode)> {
882    // Enter grant and keep it open until `_grant_open` goes out of scope.
883    let mut layout = match enter_grant_kernel_managed(process, driver_num) {
884        Ok(val) => val,
885        Err(e) => return Err((buffer, e)),
886    };
887
888    // Create the saved allow ro slice from the grant memory.
889    //
890    // # Safety
891    //
892    // This is safe because of how the grant was initially allocated and that
893    // because we were able to enter the grant the grant region must be valid
894    // and initialized. We are also holding the grant open until _grant_open
895    // goes out of scope.
896    let saved_allow_ro_slice = layout.get_allow_ro_slice();
897
898    // Index into the saved slice to get the old value. Use .get in case
899    // userspace passed us a bad allow number.
900    match saved_allow_ro_slice.get_mut(allow_num) {
901        Some(saved) => {
902            // # Safety
903            //
904            // The pointer has already been validated to be within application
905            // memory before storing the values in the saved slice.
906            let old_allow =
907                unsafe { ReadOnlyProcessBuffer::new(saved.ptr, saved.len, process.processid()) };
908
909            // Replace old values with current buffer.
910            let (ptr, len) = buffer.consume();
911            saved.ptr = ptr;
912            saved.len = len;
913
914            // Success!
915            Ok(old_allow)
916        }
917        None => Err((buffer, ErrorCode::NOSUPPORT)),
918    }
919}
920
921/// Stores the specified read-write process buffer in the kernel managed grant
922/// region for this process and driver. The previous read-write process buffer
923/// stored at the same allow_num id is returned.
924pub(crate) fn allow_rw(
925    process: &dyn Process,
926    driver_num: usize,
927    allow_num: usize,
928    buffer: ReadWriteProcessBuffer,
929) -> Result<ReadWriteProcessBuffer, (ReadWriteProcessBuffer, ErrorCode)> {
930    // Enter grant and keep it open until `_grant_open` goes out of scope.
931    let mut layout = match enter_grant_kernel_managed(process, driver_num) {
932        Ok(val) => val,
933        Err(e) => return Err((buffer, e)),
934    };
935
936    // Create the saved allow rw slice from the grant memory.
937    //
938    // # Safety
939    //
940    // This is safe because of how the grant was initially allocated and that
941    // because we were able to enter the grant the grant region must be valid
942    // and initialized. We are also holding the grant open until `_grant_open`
943    // goes out of scope.
944    let saved_allow_rw_slice = layout.get_allow_rw_slice();
945
946    // Index into the saved slice to get the old value. Use .get in case
947    // userspace passed us a bad allow number.
948    match saved_allow_rw_slice.get_mut(allow_num) {
949        Some(saved) => {
950            // # Safety
951            //
952            // The pointer has already been validated to be within application
953            // memory before storing the values in the saved slice.
954            let old_allow =
955                unsafe { ReadWriteProcessBuffer::new(saved.ptr, saved.len, process.processid()) };
956
957            // Replace old values with current buffer.
958            let (ptr, len) = buffer.consume();
959            saved.ptr = ptr;
960            saved.len = len;
961
962            // Success!
963            Ok(old_allow)
964        }
965        None => Err((buffer, ErrorCode::NOSUPPORT)),
966    }
967}
968
969/// An instance of a grant allocated for a particular process.
970///
971/// [`ProcessGrant`] is a handle to an instance of a grant that has been
972/// allocated in a specific process's grant region. A [`ProcessGrant`]
973/// guarantees that the memory for the grant has been allocated in the process's
974/// memory.
975///
976/// This is created from a [`Grant`] when that grant is entered for a specific
977/// process.
978pub struct ProcessGrant<
979    'a,
980    T: 'a,
981    Upcalls: UpcallSize,
982    AllowROs: AllowRoSize,
983    AllowRWs: AllowRwSize,
984> {
985    /// The process the grant is applied to.
986    ///
987    /// We use a reference here because instances of [`ProcessGrant`] are very
988    /// short lived. They only exist while a [`Grant`] is being entered, so we
989    /// can be sure the process still exists while a `ProcessGrant` exists. No
990    /// [`ProcessGrant`] can be stored.
991    process: &'a dyn Process,
992
993    /// The syscall driver number this grant is associated with.
994    driver_num: usize,
995
996    /// The identifier of the Grant this is applied for.
997    grant_num: usize,
998
999    /// Used to store Rust types for grant.
1000    _phantom: PhantomData<(T, Upcalls, AllowROs, AllowRWs)>,
1001}
1002
1003impl<'a, T: Default, Upcalls: UpcallSize, AllowROs: AllowRoSize, AllowRWs: AllowRwSize>
1004    ProcessGrant<'a, T, Upcalls, AllowROs, AllowRWs>
1005{
1006    /// Create a [`ProcessGrant`] for the given Grant in the given Process's
1007    /// grant region.
1008    ///
1009    /// If the grant in this process has not been setup before this will attempt
1010    /// to allocate the memory from the process's grant region.
1011    ///
1012    /// # Return
1013    ///
1014    /// If the grant is already allocated or could be allocated, and the process
1015    /// is valid, this returns `Ok(ProcessGrant)`. Otherwise it returns a
1016    /// relevant error.
1017    fn new(
1018        grant: &Grant<T, Upcalls, AllowROs, AllowRWs>,
1019        processid: ProcessId,
1020    ) -> Result<Self, Error> {
1021        // Moves non-generic code from new() into non-generic function to reduce
1022        // code bloat from the generic function being monomorphized, as it is
1023        // common to have over 50 copies of Grant::enter() in a Tock kernel (and
1024        // thus 50+ copies of this function). The returned Option indicates if
1025        // the returned pointer still needs to be initialized (in the case where
1026        // the grant was only just allocated).
1027        fn new_inner<'a>(
1028            grant_num: usize,
1029            driver_num: usize,
1030            grant_t_size: GrantDataSize,
1031            grant_t_align: GrantDataAlign,
1032            num_upcalls: UpcallItems,
1033            num_allow_ros: AllowRoItems,
1034            num_allow_rws: AllowRwItems,
1035            processid: ProcessId,
1036        ) -> Result<(Option<NonNull<u8>>, &'a dyn Process), Error> {
1037            // Here is an example of how the grants are laid out in the grant
1038            // region of process's memory:
1039            //
1040            // Mem. Addr.
1041            // 0x0040000  ┌────────────────────────────────────
1042            //            │   DriverNumN    [0x1]
1043            //            │   GrantPointerN [0x003FFC8]
1044            //            │   ...
1045            //            │   DriverNum1    [0x60000]
1046            //            │   GrantPointer1 [0x003FFC0]
1047            //            │   DriverNum0
1048            //            │   GrantPointer0 [0x0000000 (NULL)]
1049            //            ├────────────────────────────────────
1050            //            │   Process Control Block
1051            // 0x003FFE0  ├────────────────────────────────────  Grant Region ┐
1052            //            │   GrantDataN                                      │
1053            // 0x003FFC8  ├────────────────────────────────────               │
1054            //            │   GrantData1                                      ▼
1055            // 0x003FF20  ├────────────────────────────────────
1056            //            │
1057            //            │   --unallocated--
1058            //            │
1059            //            └────────────────────────────────────
1060            //
1061            // An array of pointers (one per possible grant region) point to
1062            // where the actual grant memory is allocated inside of the process.
1063            // The grant memory is not allocated until the actual grant region
1064            // is actually used.
1065
1066            let process = processid
1067                .kernel
1068                .get_process(processid)
1069                .ok_or(Error::NoSuchApp)?;
1070
1071            // Check if the grant is allocated. If not, we allocate it process
1072            // memory first. We then create an `ProcessGrant` object for this
1073            // grant.
1074            if let Some(is_allocated) = process.grant_is_allocated(grant_num) {
1075                if !is_allocated {
1076                    // Calculate the alignment and size for entire grant region.
1077                    let alloc_align = EnteredGrantKernelManagedLayout::grant_align(grant_t_align);
1078                    let alloc_size = EnteredGrantKernelManagedLayout::grant_size(
1079                        num_upcalls,
1080                        num_allow_ros,
1081                        num_allow_rws,
1082                        grant_t_size,
1083                        grant_t_align,
1084                    );
1085
1086                    // Allocate grant, the memory is still uninitialized though.
1087                    if process
1088                        .allocate_grant(grant_num, driver_num, alloc_size, alloc_align)
1089                        .is_err()
1090                    {
1091                        return Err(Error::OutOfMemory);
1092                    }
1093
1094                    let grant_ptr = process.enter_grant(grant_num)?;
1095
1096                    // Create a layout from the counts we have and initialize
1097                    // all memory so it is valid in the future to read as a
1098                    // reference.
1099                    //
1100                    // # Safety
1101                    //
1102                    // - The grant base pointer is well aligned, yet does not
1103                    //   have initialized data.
1104                    // - The pointer points to a large enough space to correctly
1105                    //   write to is guaranteed by alloc of size
1106                    //   `EnteredGrantKernelManagedLayout::grant_size`.
1107                    // - There are no proper rust references that map to these
1108                    //   addresses.
1109                    unsafe {
1110                        let _layout = EnteredGrantKernelManagedLayout::initialize_from_counts(
1111                            grant_ptr,
1112                            num_upcalls,
1113                            num_allow_ros,
1114                            num_allow_rws,
1115                            process,
1116                            grant_num,
1117                        );
1118                    }
1119
1120                    // # Safety
1121                    //
1122                    // The grant pointer points to an alloc that is alloc_size
1123                    // large and is at least as aligned as grant_t_align.
1124                    unsafe {
1125                        Ok((
1126                            Some(EnteredGrantKernelManagedLayout::offset_of_grant_data_t(
1127                                grant_ptr,
1128                                alloc_size,
1129                                grant_t_size,
1130                            )),
1131                            process,
1132                        ))
1133                    }
1134                } else {
1135                    // T was already allocated, outer function should not
1136                    // initialize T.
1137                    Ok((None, process))
1138                }
1139            } else {
1140                // Cannot use the grant region in any way if the process is
1141                // inactive.
1142                Err(Error::InactiveApp)
1143            }
1144        }
1145
1146        // Handle the bulk of the work in a function which is not templated.
1147        let (opt_raw_grant_ptr_nn, process) = new_inner(
1148            grant.grant_num,
1149            grant.driver_num,
1150            GrantDataSize(size_of::<T>()),
1151            GrantDataAlign(align_of::<T>()),
1152            UpcallItems(Upcalls::COUNT),
1153            AllowRoItems(AllowROs::COUNT),
1154            AllowRwItems(AllowRWs::COUNT),
1155            processid,
1156        )?;
1157
1158        // We can now do the initialization of T object if necessary.
1159        if let Some(allocated_ptr) = opt_raw_grant_ptr_nn {
1160            // Grant type T
1161            //
1162            // # Safety
1163            //
1164            // This is safe because:
1165            //
1166            // 1. The pointer address is valid. The pointer is allocated
1167            //    statically in process memory, and will exist for as long
1168            //    as the process does. The grant is only accessible while
1169            //    the process is still valid.
1170            //
1171            // 2. The pointer is correctly aligned. The newly allocated
1172            //    grant is aligned for type T, and there is padding inserted
1173            //    between the upcall array and the T object such that the T
1174            //    object starts a multiple of `align_of<T>` from the
1175            //    beginning of the allocation.
1176            unsafe {
1177                // Convert untyped `*mut u8` allocation to allocated type.
1178                let new_region = NonNull::cast::<T>(allocated_ptr);
1179                // We use `ptr::write` to avoid `Drop`ping the uninitialized
1180                // memory in case `T` implements the `Drop` trait.
1181                write(new_region.as_ptr(), T::default());
1182            }
1183        }
1184
1185        // We have ensured the grant is already allocated or was just allocated,
1186        // so we can create and return the `ProcessGrant` type.
1187        Ok(ProcessGrant {
1188            process,
1189            driver_num: grant.driver_num,
1190            grant_num: grant.grant_num,
1191            _phantom: PhantomData,
1192        })
1193    }
1194
1195    /// Return a [`ProcessGrant`] for a grant in a process if the process is
1196    /// valid and that process grant has already been allocated, or `None`
1197    /// otherwise.
1198    fn new_if_allocated(
1199        grant: &Grant<T, Upcalls, AllowROs, AllowRWs>,
1200        process: &'a dyn Process,
1201    ) -> Option<Self> {
1202        if let Some(is_allocated) = process.grant_is_allocated(grant.grant_num) {
1203            if is_allocated {
1204                Some(ProcessGrant {
1205                    process,
1206                    driver_num: grant.driver_num,
1207                    grant_num: grant.grant_num,
1208                    _phantom: PhantomData,
1209                })
1210            } else {
1211                // Grant has not been allocated.
1212                None
1213            }
1214        } else {
1215            // Process is invalid.
1216            None
1217        }
1218    }
1219
1220    /// Return the [`ProcessId`] of the process this [`ProcessGrant`] is
1221    /// associated with.
1222    pub fn processid(&self) -> ProcessId {
1223        self.process.processid()
1224    }
1225
1226    /// Run a function with access to the memory in the related process for the
1227    /// related Grant. This also provides access to any associated Upcalls and
1228    /// allowed buffers stored with the grant.
1229    ///
1230    /// This is "entering" the grant region, and the _only_ time when the
1231    /// contents of a grant region can be accessed.
1232    ///
1233    /// Note, a grant can only be entered once at a time. Attempting to call
1234    /// `.enter()` on a grant while it is already entered will result in a
1235    /// `panic!()`. See the comment in `access_grant()` for more information.
1236    pub fn enter<F, R>(self, fun: F) -> R
1237    where
1238        F: FnOnce(&mut GrantData<T>, &GrantKernelData) -> R,
1239    {
1240        // # `unwrap()` Safety
1241        //
1242        // `access_grant()` can only return `None` if the grant is already
1243        // entered. Since we are asking for a panic!() if the grant is entered,
1244        // `access_grant()` function will never return `None`.
1245        self.access_grant(fun, true).unwrap()
1246    }
1247
1248    /// Run a function with access to the data in the related process for the
1249    /// related Grant only if that grant region is not already entered. If the
1250    /// grant is already entered silently skip it. Also provide access to
1251    /// associated Upcalls.
1252    ///
1253    /// **You almost certainly should use `.enter()` rather than
1254    /// `.try_enter()`.**
1255    ///
1256    /// While the `.enter()` version can panic, that panic likely indicates a
1257    /// bug in the code and not a condition that should be handled. For example,
1258    /// this benign looking code is wrong:
1259    ///
1260    /// ```ignore
1261    /// self.apps.enter(thisapp, |app_grant, _| {
1262    ///     // Update state in the grant region of `thisapp`. Also, mark that
1263    ///     // `thisapp` needs to run again.
1264    ///     app_grant.runnable = true;
1265    ///
1266    ///     // Now, check all apps to see if any are ready to run.
1267    ///     let mut work_left_to_do = false;
1268    ///     self.apps.iter().each(|other_app| {
1269    ///         other_app.enter(|other_app_grant, _| { // ERROR! This leads to a
1270    ///             if other_app_grant.runnable {      // grant being entered
1271    ///                 work_left_to_do = true;        // twice!
1272    ///             }
1273    ///         })
1274    ///     })
1275    /// })
1276    /// ```
1277    ///
1278    /// The example is wrong because it tries to iterate across all grant
1279    /// regions while one of them is already entered. This will lead to a grant
1280    /// region being entered twice which violates Rust's memory restrictions and
1281    /// is undefined behavior.
1282    ///
1283    /// However, since the example uses `.enter()` on the iteration, Tock will
1284    /// panic when the grant is entered for the second time, notifying the
1285    /// developer that something is wrong. The fix is to exit out of the first
1286    /// `.enter()` before attempting to iterate over the grant for all
1287    /// processes.
1288    ///
1289    /// However, if the example used `.try_enter()` in the iter loop, there
1290    /// would be no panic, but the already entered grant would be silently
1291    /// skipped. This can hide subtle bugs if the skipped grant is only relevant
1292    /// in certain cases.
1293    ///
1294    /// Therefore, only use `try_enter()` if you are sure you want to skip the
1295    /// already entered grant. Cases for this are rare.
1296    ///
1297    /// ## Return
1298    ///
1299    /// Returns `None` if the grant is already entered. Otherwise returns
1300    /// `Some(fun())`.
1301    pub fn try_enter<F, R>(self, fun: F) -> Option<R>
1302    where
1303        F: FnOnce(&mut GrantData<T>, &GrantKernelData) -> R,
1304    {
1305        self.access_grant(fun, false)
1306    }
1307
1308    /// Run a function with access to the memory in the related process for the
1309    /// related Grant. Also provide this function with access to any associated
1310    /// Upcalls and an allocator for allocating additional memory in the
1311    /// process's grant region.
1312    ///
1313    /// This is "entering" the grant region, and the _only_ time when the
1314    /// contents of a grant region can be accessed.
1315    ///
1316    /// Note, a grant can only be entered once at a time. Attempting to call
1317    /// `.enter()` on a grant while it is already entered will result in a
1318    /// panic!()`. See the comment in `access_grant()` for more information.
1319    pub fn enter_with_allocator<F, R>(self, fun: F) -> R
1320    where
1321        F: FnOnce(&mut GrantData<T>, &GrantKernelData, &mut GrantRegionAllocator) -> R,
1322    {
1323        // # `unwrap()` Safety
1324        //
1325        // `access_grant()` can only return `None` if the grant is already
1326        // entered. Since we are asking for a panic!() if the grant is entered,
1327        // `access_grant()` function will never return `None`.
1328        self.access_grant_with_allocator(fun, true).unwrap()
1329    }
1330
1331    /// Access the [`ProcessGrant`] memory and run a closure on the process's
1332    /// grant memory.
1333    ///
1334    /// If `panic_on_reenter` is `true`, this will panic if the grant region is
1335    /// already currently entered. If `panic_on_reenter` is `false`, this will
1336    /// return `None` if the grant region is entered and do nothing.
1337    fn access_grant<F, R>(self, fun: F, panic_on_reenter: bool) -> Option<R>
1338    where
1339        F: FnOnce(&mut GrantData<T>, &GrantKernelData) -> R,
1340    {
1341        self.access_grant_with_allocator(
1342            |grant_data, kernel_data, _allocator| fun(grant_data, kernel_data),
1343            panic_on_reenter,
1344        )
1345    }
1346
1347    /// Access the [`ProcessGrant`] memory and run a closure on the process's
1348    /// grant memory.
1349    ///
1350    /// If `panic_on_reenter` is `true`, this will panic if the grant region is
1351    /// already currently entered. If `panic_on_reenter` is `false`, this will
1352    /// return `None` if the grant region is entered and do nothing.
1353    fn access_grant_with_allocator<F, R>(self, fun: F, panic_on_reenter: bool) -> Option<R>
1354    where
1355        F: FnOnce(&mut GrantData<T>, &GrantKernelData, &mut GrantRegionAllocator) -> R,
1356    {
1357        // Access the grant that is in process memory. This can only fail if
1358        // the grant is already entered.
1359        let grant_ptr = self
1360            .process
1361            .enter_grant(self.grant_num)
1362            .map_err(|_err| {
1363                // If we get an error it is because the grant is already
1364                // entered. `process.enter_grant()` can fail for several
1365                // reasons, but only the double enter case can happen once a
1366                // grant has been applied. The other errors would be detected
1367                // earlier (i.e. before the grant can be applied).
1368
1369                // If `panic_on_reenter` is false, we skip this error and do
1370                // nothing with this grant.
1371                if !panic_on_reenter {
1372                    return;
1373                }
1374
1375                // If `enter_grant` fails, we panic!() to notify the developer
1376                // that they tried to enter the same grant twice which is
1377                // prohibited because it would result in two mutable references
1378                // existing for the same memory. This preserves type correctness
1379                // (but does crash the system).
1380                //
1381                // ## Explanation and Rationale
1382                //
1383                // This panic represents a tradeoff. While it is undesirable to
1384                // have the potential for a runtime crash in this grant region
1385                // code, it balances usability with type correctness. The
1386                // challenge is that calling `self.apps.iter()` is a common
1387                // pattern in capsules to access the grant region of every app
1388                // that is using the capsule, and sometimes it is intuitive to
1389                // call that inside of a `self.apps.enter(processid, |app| {...})`
1390                // closure. However, `.enter()` means that app's grant region is
1391                // entered, and then a naive `.iter()` would re-enter the grant
1392                // region and cause undefined behavior. We considered different
1393                // options to resolve this.
1394                //
1395                // 1. Have `.iter()` only iterate over grant regions which are
1396                //    not entered. This avoids the bug, but could lead to
1397                //    unexpected behavior, as `self.apps.iter()` will do
1398                //    different things depending on where in a capsule it is
1399                //    called.
1400                // 2. Have the compiler detect when `.iter()` is called when a
1401                //    grant region has already been entered. We don't know of a
1402                //    viable way to implement this.
1403                // 3. Panic if `.iter()` is called when a grant is already
1404                //    entered.
1405                //
1406                // We decided on option 3 because it balances minimizing
1407                // surprises (`self.apps.iter()` will always iterate all grants)
1408                // while also protecting against the bug. We expect that any
1409                // code that attempts to call `self.apps.iter()` after calling
1410                // `.enter()` will immediately encounter this `panic!()` and
1411                // have to be refactored before any tests will be successful.
1412                // Therefore, this `panic!()` should only occur at
1413                // development/testing time.
1414                //
1415                // ## How to fix this error
1416                //
1417                // If you are seeing this panic, you need to refactor your
1418                // capsule to not call `.iter()` or `.each()` from inside a
1419                // `.enter()` closure. That is, you need to close the grant
1420                // region you are currently in before trying to iterate over all
1421                // grant regions.
1422                panic!("Attempted to re-enter a grant region.");
1423            })
1424            .ok()?;
1425        let grant_t_align = GrantDataAlign(align_of::<T>());
1426        let grant_t_size = GrantDataSize(size_of::<T>());
1427
1428        let alloc_size = EnteredGrantKernelManagedLayout::grant_size(
1429            UpcallItems(Upcalls::COUNT),
1430            AllowRoItems(AllowROs::COUNT),
1431            AllowRwItems(AllowRWs::COUNT),
1432            grant_t_size,
1433            grant_t_align,
1434        );
1435
1436        // Parse layout of entire grant allocation using the known base pointer.
1437        //
1438        // # Safety
1439        //
1440        // Grant pointer is well aligned and points to initialized data.
1441        let layout = unsafe {
1442            EnteredGrantKernelManagedLayout::read_from_base(grant_ptr, self.process, self.grant_num)
1443        };
1444
1445        // Get references to all of the saved upcall data.
1446        //
1447        // # Safety
1448        //
1449        // - Pointer is well aligned and initialized with data from Self::new()
1450        //   call.
1451        // - Data will not be modified externally while this immutable reference
1452        //   is alive.
1453        // - Data is accessible for the entire duration of this immutable
1454        //   reference.
1455        // - No other mutable reference to this memory exists concurrently.
1456        //   Mutable reference to this memory are only created through the
1457        //   kernel in the syscall interface which is serialized in time with
1458        //   this call.
1459        let (saved_upcalls_slice, saved_allow_ro_slice, saved_allow_rw_slice) =
1460            layout.get_resource_slices();
1461        let grant_data = unsafe {
1462            EnteredGrantKernelManagedLayout::offset_of_grant_data_t(
1463                grant_ptr,
1464                alloc_size,
1465                grant_t_size,
1466            )
1467            .cast()
1468            .as_mut()
1469        };
1470
1471        // Create a wrapped objects that are passed to functor.
1472        let mut grant_data = GrantData::new(grant_data);
1473        let kernel_data = GrantKernelData::new(
1474            saved_upcalls_slice,
1475            saved_allow_ro_slice,
1476            saved_allow_rw_slice,
1477            self.driver_num,
1478            self.process,
1479        );
1480        // Setup an allocator in case the capsule needs additional memory in the
1481        // grant space.
1482        let mut allocator = GrantRegionAllocator {
1483            processid: self.process.processid(),
1484        };
1485
1486        // Call functor and pass back value.
1487        Some(fun(&mut grant_data, &kernel_data, &mut allocator))
1488    }
1489}
1490
1491/// Grant which was allocated from the kernel-owned grant region in a specific
1492/// process's memory, separately from a normal `Grant`.
1493///
1494/// A [`CustomGrant`] allows a capsule to allocate additional memory on behalf
1495/// of a process.
1496pub struct CustomGrant<T> {
1497    /// An identifier for this custom grant within a process's grant region.
1498    ///
1499    /// Here, this is an opaque reference that Process uses to access the
1500    /// custom grant allocation. This setup ensures that Process owns the grant
1501    /// memory.
1502    identifier: ProcessCustomGrantIdentifier,
1503
1504    /// Identifier for the process where this custom grant is allocated.
1505    processid: ProcessId,
1506
1507    /// Used to keep the Rust type of the grant.
1508    _phantom: PhantomData<T>,
1509}
1510
1511impl<T> CustomGrant<T> {
1512    /// Creates a new [`CustomGrant`].
1513    fn new(identifier: ProcessCustomGrantIdentifier, processid: ProcessId) -> Self {
1514        CustomGrant {
1515            identifier,
1516            processid,
1517            _phantom: PhantomData,
1518        }
1519    }
1520
1521    /// Helper function to get the [`ProcessId`] from the custom grant.
1522    pub fn processid(&self) -> ProcessId {
1523        self.processid
1524    }
1525
1526    /// Gives access to inner data within the given closure.
1527    ///
1528    /// If the process has since been restarted or crashed, or the memory is
1529    /// otherwise no longer present, then this function will not call the given
1530    /// closure, and will instead directly return `Err(Error::NoSuchApp)`.
1531    ///
1532    /// Because this function requires `&mut self`, it should be impossible to
1533    /// access the inner data of a given `CustomGrant` reentrantly. Thus the
1534    /// reentrance detection we use for non-custom grants is not needed here.
1535    pub fn enter<F, R>(&self, fun: F) -> Result<R, Error>
1536    where
1537        F: FnOnce(GrantData<'_, T>) -> R,
1538    {
1539        // Verify that the process this CustomGrant was allocated within still
1540        // exists.
1541        self.processid
1542            .kernel
1543            .process_map_or(Err(Error::NoSuchApp), self.processid, |process| {
1544                // App is valid.
1545
1546                // Now try to access the custom grant memory.
1547                let grant_ptr = process.enter_custom_grant(self.identifier)?;
1548
1549                // # Safety
1550                //
1551                // `grant_ptr` must be a valid pointer and there must not exist
1552                // any other references to the same memory. We verify the
1553                // pointer is valid and aligned when the memory is allocated and
1554                // `CustomGrant` is created. We are sure that there are no
1555                // other references because the only way to create a reference
1556                // is using this `enter()` function, and it can only be called
1557                // once (because of the `&mut self` requirement).
1558                let custom_grant = unsafe { &mut *(grant_ptr as *mut T) };
1559                let borrowed = GrantData::new(custom_grant);
1560                Ok(fun(borrowed))
1561            })
1562    }
1563}
1564
1565/// Tool for allocating additional memory regions in a process's grant region.
1566///
1567/// This is optionally provided along with a grant so that if a capsule needs
1568/// per-process dynamic allocation it can allocate additional memory.
1569pub struct GrantRegionAllocator {
1570    /// The process the allocator will allocate memory from.
1571    processid: ProcessId,
1572}
1573
1574impl GrantRegionAllocator {
1575    /// Allocates a new [`CustomGrant`] initialized using the given closure.
1576    ///
1577    /// The closure will be called exactly once, and the result will be used to
1578    /// initialize the owned value.
1579    ///
1580    /// This interface was chosen instead of a simple `alloc(val)` as it's
1581    /// much more likely to optimize out all stack intermediates. This
1582    /// helps to prevent stack overflows when allocating large values.
1583    ///
1584    /// # Panic Safety
1585    ///
1586    /// If `init` panics, the freshly allocated memory may leak.
1587    pub fn alloc_with<T, F>(&self, init: F) -> Result<CustomGrant<T>, Error>
1588    where
1589        F: FnOnce() -> T,
1590    {
1591        let (custom_grant_identifier, typed_ptr) = self.alloc_raw::<T>()?;
1592
1593        // # Safety
1594        //
1595        // Writing to this pointer is safe as long as the pointer is valid
1596        // and aligned. `alloc_raw()` guarantees these constraints are met.
1597        unsafe {
1598            // We use `ptr::write` to avoid `Drop`ping the uninitialized memory
1599            // in case `T` implements the `Drop` trait.
1600            write(typed_ptr.as_ptr(), init());
1601        }
1602
1603        Ok(CustomGrant::new(custom_grant_identifier, self.processid))
1604    }
1605
1606    /// Allocates a slice of n instances of a given type. Each instance is
1607    /// initialized using the provided function.
1608    ///
1609    /// The provided function will be called exactly `n` times, and will be
1610    /// passed the index it's initializing, from `0` through `NUM_ITEMS - 1`.
1611    ///
1612    /// # Panic Safety
1613    ///
1614    /// If `val_func` panics, the freshly allocated memory and any values
1615    /// already written will be leaked.
1616    pub fn alloc_n_with<T, F, const NUM_ITEMS: usize>(
1617        &self,
1618        mut init: F,
1619    ) -> Result<CustomGrant<[T; NUM_ITEMS]>, Error>
1620    where
1621        F: FnMut(usize) -> T,
1622    {
1623        let (custom_grant_identifier, typed_ptr) = self.alloc_n_raw::<T>(NUM_ITEMS)?;
1624
1625        for i in 0..NUM_ITEMS {
1626            // # Safety
1627            //
1628            // The allocate function guarantees that `ptr` points to memory
1629            // large enough to allocate `num_items` copies of the object.
1630            unsafe {
1631                write(typed_ptr.as_ptr().add(i), init(i));
1632            }
1633        }
1634
1635        Ok(CustomGrant::new(custom_grant_identifier, self.processid))
1636    }
1637
1638    /// Allocates uninitialized grant memory appropriate to store a `T`.
1639    ///
1640    /// The caller must initialize the memory.
1641    ///
1642    /// Also returns a ProcessCustomGrantIdentifier to access the memory later.
1643    fn alloc_raw<T>(&self) -> Result<(ProcessCustomGrantIdentifier, NonNull<T>), Error> {
1644        self.alloc_n_raw::<T>(1)
1645    }
1646
1647    /// Allocates space for a dynamic number of items.
1648    ///
1649    /// The caller is responsible for initializing the returned memory.
1650    ///
1651    /// Returns memory appropriate for storing `num_items` contiguous instances
1652    /// of `T` and a ProcessCustomGrantIdentifier to access the memory later.
1653    fn alloc_n_raw<T>(
1654        &self,
1655        num_items: usize,
1656    ) -> Result<(ProcessCustomGrantIdentifier, NonNull<T>), Error> {
1657        let (custom_grant_identifier, raw_ptr) =
1658            self.alloc_n_raw_inner(num_items, size_of::<T>(), align_of::<T>())?;
1659        let typed_ptr = NonNull::cast::<T>(raw_ptr);
1660
1661        Ok((custom_grant_identifier, typed_ptr))
1662    }
1663
1664    /// Helper to reduce code bloat by avoiding monomorphization.
1665    fn alloc_n_raw_inner(
1666        &self,
1667        num_items: usize,
1668        single_alloc_size: usize,
1669        alloc_align: usize,
1670    ) -> Result<(ProcessCustomGrantIdentifier, NonNull<u8>), Error> {
1671        let alloc_size = single_alloc_size
1672            .checked_mul(num_items)
1673            .ok_or(Error::OutOfMemory)?;
1674        self.processid
1675            .kernel
1676            .process_map_or(Err(Error::NoSuchApp), self.processid, |process| {
1677                process
1678                    .allocate_custom_grant(alloc_size, alloc_align)
1679                    .map_or(
1680                        Err(Error::OutOfMemory),
1681                        |(custom_grant_identifier, raw_ptr)| Ok((custom_grant_identifier, raw_ptr)),
1682                    )
1683            })
1684    }
1685}
1686
1687/// Type for storing an object of type T in process memory that is only
1688/// accessible by the kernel.
1689///
1690/// A single [`Grant`] can allocate space for one object of type T for each
1691/// process on the board. Each allocated object will reside in the grant region
1692/// belonging to the process that the object is allocated for. The [`Grant`]
1693/// type is used to get access to [`ProcessGrant`]s, which are tied to a
1694/// specific process and provide access to the memory object allocated for that
1695/// process.
1696pub struct Grant<T: Default, Upcalls: UpcallSize, AllowROs: AllowRoSize, AllowRWs: AllowRwSize> {
1697    /// Hold a reference to the core kernel so we can iterate processes.
1698    pub(crate) kernel: &'static Kernel,
1699
1700    /// Keep track of the syscall driver number assigned to the capsule that is
1701    /// using this grant. This allows us to uniquely identify upcalls stored in
1702    /// this grant.
1703    driver_num: usize,
1704
1705    /// The identifier for this grant. Having an identifier allows the Process
1706    /// implementation to lookup the memory for this grant in the specific
1707    /// process.
1708    grant_num: usize,
1709
1710    /// Used to store the Rust types for grant.
1711    ptr: PhantomData<(T, Upcalls, AllowROs, AllowRWs)>,
1712}
1713
1714impl<T: Default, Upcalls: UpcallSize, AllowROs: AllowRoSize, AllowRWs: AllowRwSize>
1715    Grant<T, Upcalls, AllowROs, AllowRWs>
1716{
1717    /// Create a new [`Grant`] type which allows a capsule to store
1718    /// process-specific data for each process in the process's memory region.
1719    ///
1720    /// This must only be called from the main kernel so that it can ensure that
1721    /// `grant_index` is a valid index.
1722    pub(crate) fn new(kernel: &'static Kernel, driver_num: usize, grant_index: usize) -> Self {
1723        Self {
1724            kernel,
1725            driver_num,
1726            grant_num: grant_index,
1727            ptr: PhantomData,
1728        }
1729    }
1730
1731    /// Enter the grant for a specific process.
1732    ///
1733    /// This creates a [`ProcessGrant`] which is a handle for a grant allocated
1734    /// for a specific process. Then, that [`ProcessGrant`] is entered and the
1735    /// provided closure is run with access to the memory in the grant region.
1736    pub fn enter<F, R>(&self, processid: ProcessId, fun: F) -> Result<R, Error>
1737    where
1738        F: FnOnce(&mut GrantData<T>, &GrantKernelData) -> R,
1739    {
1740        let pg = ProcessGrant::new(self, processid)?;
1741
1742        // If we have managed to create an `ProcessGrant`, all we need
1743        // to do is actually access the memory and run the
1744        // capsule-provided closure. This can only fail if the grant is
1745        // already entered, at which point the kernel will panic.
1746        Ok(pg.enter(fun))
1747    }
1748
1749    /// Enter the grant for a specific process with access to an allocator.
1750    ///
1751    /// This creates an [`ProcessGrant`] which is a handle for a grant allocated
1752    /// for a specific process. Then, that [`ProcessGrant`] is entered and the
1753    /// provided closure is run with access to the memory in the grant region.
1754    ///
1755    /// The allocator allows the caller to dynamically allocate additional
1756    /// memory in the process's grant region.
1757    pub fn enter_with_allocator<F, R>(&self, processid: ProcessId, fun: F) -> Result<R, Error>
1758    where
1759        F: FnOnce(&mut GrantData<T>, &GrantKernelData, &mut GrantRegionAllocator) -> R,
1760    {
1761        // Get the `ProcessGrant` for the process, possibly needing to
1762        // actually allocate the memory in the process's grant region to
1763        // do so. This can fail for a variety of reasons, and if so we
1764        // return the error to the capsule.
1765        let pg = ProcessGrant::new(self, processid)?;
1766
1767        // If we have managed to create an `ProcessGrant`, all we need
1768        // to do is actually access the memory and run the
1769        // capsule-provided closure. This can only fail if the grant is
1770        // already entered, at which point the kernel will panic.
1771        Ok(pg.enter_with_allocator(fun))
1772    }
1773
1774    /// Run a function on the grant for each active process if the grant has
1775    /// been allocated for that process.
1776    ///
1777    /// This will silently skip any process where the grant has not previously
1778    /// been allocated. This will also silently skip any invalid processes.
1779    ///
1780    /// Calling this function when an [`ProcessGrant`] for a process is
1781    /// currently entered will result in a panic.
1782    pub fn each<F>(&self, mut fun: F)
1783    where
1784        F: FnMut(ProcessId, &mut GrantData<T>, &GrantKernelData),
1785    {
1786        // Create a the iterator across `ProcessGrant`s for each process.
1787        for pg in self.iter() {
1788            let processid = pg.processid();
1789            // Since we iterating, there is no return value we need to worry
1790            // about.
1791            pg.enter(|data, upcalls| fun(processid, data, upcalls));
1792        }
1793    }
1794
1795    /// Get an iterator over all processes and their active grant regions for
1796    /// this particular grant.
1797    ///
1798    /// Calling this function when an [`ProcessGrant`] for a process is
1799    /// currently entered will result in a panic.
1800    pub fn iter(&self) -> Iter<T, Upcalls, AllowROs, AllowRWs> {
1801        Iter {
1802            grant: self,
1803            subiter: self.kernel.get_process_iter(),
1804        }
1805    }
1806}
1807
1808/// Type to iterate [`ProcessGrant`]s across processes.
1809pub struct Iter<
1810    'a,
1811    T: 'a + Default,
1812    Upcalls: UpcallSize,
1813    AllowROs: AllowRoSize,
1814    AllowRWs: AllowRwSize,
1815> {
1816    /// The grant type to use.
1817    grant: &'a Grant<T, Upcalls, AllowROs, AllowRWs>,
1818
1819    /// Iterator over valid processes.
1820    subiter: core::iter::FilterMap<
1821        core::slice::Iter<'a, Option<&'static dyn Process>>,
1822        fn(&Option<&'static dyn Process>) -> Option<&'static dyn Process>,
1823    >,
1824}
1825
1826impl<'a, T: Default, Upcalls: UpcallSize, AllowROs: AllowRoSize, AllowRWs: AllowRwSize> Iterator
1827    for Iter<'a, T, Upcalls, AllowROs, AllowRWs>
1828{
1829    type Item = ProcessGrant<'a, T, Upcalls, AllowROs, AllowRWs>;
1830
1831    fn next(&mut self) -> Option<Self::Item> {
1832        let grant = self.grant;
1833        // Get the next `ProcessId` from the kernel processes array that is
1834        // setup to use this grant. Since the iterator itself is saved calling
1835        // this function again will start where we left off.
1836        self.subiter
1837            .find_map(|process| ProcessGrant::new_if_allocated(grant, process))
1838    }
1839}