len0rd/FreeRTOS-Kernel
1
0
Fork 0
mirror of https://github.com/FreeRTOS/FreeRTOS-Kernel.git synced 2026-07-10 13:29:45 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge 045427105c into ae46383c90

Browse source
This commit is contained in:
Richard Elberger 2026-07-03 15:21:30 +08:00 • committed by GitHub
commit 360463ec65
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 2063 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

7
.github/.cSpellWords.txt vendored
View file

@ -1017,3 +1017,10 @@ XPSR
XRAM
xtal
XTENSA
ACLE
acle
APDA
APDB
REGEN
SSVE
SVCR

35
MISRA.md
View file

@ -18,6 +18,16 @@ with ( Assuming rule 8.4 violation; with justification in point 1 ):
grep 'MISRA Ref 8.4.1' . -rI
```
#### Dir 4.3
MISRA C:2012 Dir 4.3: Assembly language shall be encapsulated and isolated.
_Ref 4.3.1_
- AArch64 port files (portable/GCC/ARM_AARCH64_ARMV9/) use inline assembly
to access system registers (GIC, generic timer, PAC keys, MTE, SVE
configuration) that have no C-language equivalent. Assembly is confined to
the port layer and isolated from application code.
#### Dir 4.7
MISRA C:2012 Dir 4.7: If a function returns error information, then that error
@ -116,6 +126,31 @@ _Ref 11.5.5_
because data storage buffers are implemented as uint8_t arrays for the
ease of sizing, alignment and access.
#### Rule 5.8
MISRA C:2012 Rule 5.8: Identifiers that define objects or functions with
external linkage shall be unique.
_Ref 5.8.1_
- The Armv9 port (portable/GCC/ARM_AARCH64_ARMV9/portmacro.h) defines macros
that redirect `pvPortMalloc` and `vPortFree` to MTE-tagged wrapper functions
when `configARMV9_MTE_HEAP == 1`. This keeps MTE heap tagging logic isolated
in the port without modifying shared upstream kernel files. An alternative
that eliminates this deviation (integrating tagging into heap_4.c) is
documented in the source.
#### Rule 11.6
MISRA C:2012 Rule 11.6: A cast shall not be performed between a pointer to void
and an arithmetic type.
_Ref 11.6.1_
- The Armv9 MTE port (portable/GCC/ARM_AARCH64_ARMV9/mte_port.c) casts
between pointer and uintptr_t to strip the MTE tag bits (top byte) from
tagged pointers before passing them to the underlying allocator. This is
required because MTE tags are encoded in pointer bits 56-59 and must be
removed to recover the canonical address.
#### Rule 14.3
MISRA C-2012 Rule 14.3: Controlling expressions shall not be invariant.

4
include/FreeRTOS.h
View file

@ -2752,6 +2752,10 @@
#define portTASK_USES_FLOATING_POINT()
#endif
#ifndef portSTRIP_ADDRESS_TAG
#define portSTRIP_ADDRESS_TAG( pxPointer ) ( pxPointer )
#endif
#ifndef portALLOCATE_SECURE_CONTEXT
#define portALLOCATE_SECURE_CONTEXT( ulSecureStackSize )
#endif

72
include/stack_macros.h
View file

@ -53,6 +53,10 @@
#define portSTACK_LIMIT_PADDING 0
#endif
#ifndef portSTRIP_ADDRESS_TAG
#define portSTRIP_ADDRESS_TAG( pxPointer ) ( pxPointer )
#endif
/* Stack overflow check is not straight forward to implement for MPU ports
* because of the following reasons:
* 1. The context is stored in TCB and as a result, pxTopOfStack member points
@ -67,15 +71,15 @@
#if ( ( configCHECK_FOR_STACK_OVERFLOW == 1 ) && ( portSTACK_GROWTH < 0 ) && ( portUSING_MPU_WRAPPERS != 1 ) )
/* Only the current stack state is to be checked. */
#define taskCHECK_FOR_STACK_OVERFLOW() \
do \
{ \
/* Is the currently saved stack pointer within the stack limit? */ \
if( pxCurrentTCB->pxTopOfStack <= pxCurrentTCB->pxStack + portSTACK_LIMIT_PADDING ) \
{ \
char * pcOverflowTaskName = pxCurrentTCB->pcTaskName; \
vApplicationStackOverflowHook( ( TaskHandle_t ) pxCurrentTCB, pcOverflowTaskName ); \
} \
#define taskCHECK_FOR_STACK_OVERFLOW() \
do \
{ \
/* Is the currently saved stack pointer within the stack limit? */ \
if( portSTRIP_ADDRESS_TAG( pxCurrentTCB->pxTopOfStack ) <= pxCurrentTCB->pxStack + portSTACK_LIMIT_PADDING ) \
{ \
char * pcOverflowTaskName = pxCurrentTCB->pcTaskName; \
vApplicationStackOverflowHook( ( TaskHandle_t ) pxCurrentTCB, pcOverflowTaskName ); \
} \
} while( 0 )
#endif /* configCHECK_FOR_STACK_OVERFLOW == 1 */
@ -84,15 +88,15 @@
#if ( ( configCHECK_FOR_STACK_OVERFLOW == 1 ) && ( portSTACK_GROWTH > 0 ) && ( portUSING_MPU_WRAPPERS != 1 ) )
/* Only the current stack state is to be checked. */
#define taskCHECK_FOR_STACK_OVERFLOW() \
do \
{ \
/* Is the currently saved stack pointer within the stack limit? */ \
if( pxCurrentTCB->pxTopOfStack >= pxCurrentTCB->pxEndOfStack - portSTACK_LIMIT_PADDING ) \
{ \
char * pcOverflowTaskName = pxCurrentTCB->pcTaskName; \
vApplicationStackOverflowHook( ( TaskHandle_t ) pxCurrentTCB, pcOverflowTaskName ); \
} \
#define taskCHECK_FOR_STACK_OVERFLOW() \
do \
{ \
/* Is the currently saved stack pointer within the stack limit? */ \
if( portSTRIP_ADDRESS_TAG( pxCurrentTCB->pxTopOfStack ) >= pxCurrentTCB->pxEndOfStack - portSTACK_LIMIT_PADDING ) \
{ \
char * pcOverflowTaskName = pxCurrentTCB->pcTaskName; \
vApplicationStackOverflowHook( ( TaskHandle_t ) pxCurrentTCB, pcOverflowTaskName ); \
} \
} while( 0 )
#endif /* configCHECK_FOR_STACK_OVERFLOW == 1 */
@ -100,21 +104,21 @@
#if ( ( configCHECK_FOR_STACK_OVERFLOW > 1 ) && ( portSTACK_GROWTH < 0 ) && ( portUSING_MPU_WRAPPERS != 1 ) )
#define taskCHECK_FOR_STACK_OVERFLOW() \
do \
{ \
const uint32_t * const pulStack = ( uint32_t * ) pxCurrentTCB->pxStack; \
const uint32_t ulCheckValue = ( uint32_t ) 0xa5a5a5a5U; \
\
if( ( pxCurrentTCB->pxTopOfStack <= pxCurrentTCB->pxStack + portSTACK_LIMIT_PADDING ) || \
( pulStack[ 0 ] != ulCheckValue ) || \
( pulStack[ 1 ] != ulCheckValue ) || \
( pulStack[ 2 ] != ulCheckValue ) || \
( pulStack[ 3 ] != ulCheckValue ) ) \
{ \
char * pcOverflowTaskName = pxCurrentTCB->pcTaskName; \
vApplicationStackOverflowHook( ( TaskHandle_t ) pxCurrentTCB, pcOverflowTaskName ); \
} \
#define taskCHECK_FOR_STACK_OVERFLOW() \
do \
{ \
const uint32_t * const pulStack = ( uint32_t * ) pxCurrentTCB->pxStack; \
const uint32_t ulCheckValue = ( uint32_t ) 0xa5a5a5a5U; \
\
if( ( portSTRIP_ADDRESS_TAG( pxCurrentTCB->pxTopOfStack ) <= ( StackType_t * ) portSTRIP_ADDRESS_TAG( pxCurrentTCB->pxStack ) + portSTACK_LIMIT_PADDING ) || \
( pulStack[ 0 ] != ulCheckValue ) || \
( pulStack[ 1 ] != ulCheckValue ) || \
( pulStack[ 2 ] != ulCheckValue ) || \
( pulStack[ 3 ] != ulCheckValue ) ) \
{ \
char * pcOverflowTaskName = pxCurrentTCB->pcTaskName; \
vApplicationStackOverflowHook( ( TaskHandle_t ) pxCurrentTCB, pcOverflowTaskName ); \
} \
} while( 0 )
#endif /* #if( configCHECK_FOR_STACK_OVERFLOW > 1 ) */
@ -134,7 +138,7 @@
\
pcEndOfStack -= sizeof( ucExpectedStackBytes ); \
\
if( ( pxCurrentTCB->pxTopOfStack >= pxCurrentTCB->pxEndOfStack - portSTACK_LIMIT_PADDING ) || \
if( ( portSTRIP_ADDRESS_TAG( pxCurrentTCB->pxTopOfStack ) >= pxCurrentTCB->pxEndOfStack - portSTACK_LIMIT_PADDING ) || \
( memcmp( ( void * ) pcEndOfStack, ( void * ) ucExpectedStackBytes, sizeof( ucExpectedStackBytes ) ) != 0 ) ) \
{ \
char * pcOverflowTaskName = pxCurrentTCB->pcTaskName; \

9
portable/CMakeLists.txt
View file

@ -75,6 +75,12 @@ add_library(freertos_kernel_port OBJECT
GCC/ARM_AARCH64_SRE/port.c
GCC/ARM_AARCH64_SRE/portASM.S>
# ARMv9-A port for GCC (SVE2, MTE, PAC, BTI)
$<$<STREQUAL:${FREERTOS_PORT},GCC_ARM_AARCH64_ARMV9>:
GCC/ARM_AARCH64_ARMV9/port.c
GCC/ARM_AARCH64_ARMV9/mte_port.c
GCC/ARM_AARCH64_ARMV9/portASM.S>
# ARMv6-M port for GCC
$<$<STREQUAL:${FREERTOS_PORT},GCC_ARM_CM0>:
GCC/ARM_CM0/port.c
@ -1014,6 +1020,9 @@ target_include_directories(freertos_kernel_port_headers INTERFACE
$<$<STREQUAL:${FREERTOS_PORT},GCC_ARM_AARCH64>:${CMAKE_CURRENT_LIST_DIR}/GCC/ARM_AARCH64>
$<$<STREQUAL:${FREERTOS_PORT},GCC_ARM_AARCH64_SRE>:${CMAKE_CURRENT_LIST_DIR}/GCC/ARM_AARCH64_SRE>
# ARMv9-A port for GCC
$<$<STREQUAL:${FREERTOS_PORT},GCC_ARM_AARCH64_ARMV9>:${CMAKE_CURRENT_LIST_DIR}/GCC/ARM_AARCH64_ARMV9>
# ARMv6-M port for GCC
$<$<STREQUAL:${FREERTOS_PORT},GCC_ARM_CM0>:${CMAKE_CURRENT_LIST_DIR}/GCC/ARM_CM0>

79
portable/GCC/ARM_AARCH64_ARMV9/README.md Normal file
View file

@ -0,0 +1,79 @@
# Armv9-A Architecture Port
This port extends the AArch64 SRE (System Register Enable) port with Armv9-A
architectural features. It targets processors implementing FEAT_SVE2, FEAT_MTE2,
FEAT_PAuth, FEAT_BTI, and FEAT_RME.
## Relationship to ARM_AARCH64_SRE
This port is based on `ARM_AARCH64_SRE` and shares the same GICv3 system
register interface and generic timer infrastructure. The Armv9 additions are
guarded by `configARMV9_*` configuration flags — when all flags are disabled
(default), this port behaves identically to the Armv8 SRE port.
**Note:** This port is currently maintained as a separate directory from
`ARM_AARCH64_SRE`. A future decision will determine whether to fold these
changes into the existing SRE port (behind `configARMV9_*` guards) or maintain
them as a distinct port. Until that decision is made, this directory exists
independently to avoid regression risk to the Armv8 user base.
## Configuration Flags
All flags default to 0 (disabled). Set in `FreeRTOSConfig.h`:
| Flag | Feature |
|------|---------|
| `configARMV9_SVE2` | SVE2 register save/restore (Z0-Z31, P0-P15, FFR) |
| `configARMV9_SSVE` | Streaming SVE mode (SVCR save/restore) |
| `configARMV9_TASK_VL` | Per-task vector length (ZCR_EL1) |
| `configARMV9_PAC` | Per-task PAC key loading |
| `configARMV9_PAC_FRAME` | PAC-signed saved context frame |
| `configARMV9_MTE_HEAP` | MTE-tagged heap allocations |
| `configARMV9_MTE_STACK` | MTE-tagged task stacks |
| `configARMV9_BTI` | BTI landing pads in port assembly |
| `configARMV9_RME` | Realm Management Extension |
| `configARMV9_RME_EL2` | EL2 hypervisor operation |
## Hardware Prerequisites
- `CPACR_EL1.ZEN = 3` (SVE access from EL1/EL0)
- `CPTR_EL3.EZ = 1` (no SVE trap from EL3)
- `ZCR_EL2` / `ZCR_EL1` configured for desired vector length
- `SCTLR_EL1.EnIA = 1` (PAC enabled)
- `SCR_EL3.API/APK = 1`, `HCR_EL2.API/APK = 1` (PAC not trapped)
- `SCTLR_EL1.ATA = 1` (MTE enabled)
- `TCR_EL1.TBI0 = 1` (Top Byte Ignore for MTE pointers)
- MMU with GP bit (BTI) and Tagged Normal memory (MTE)
## Files
| File | Description |
|------|-------------|
| `port.c` | Task initialization, tick handler, critical sections |
| `portASM.S` | Context switch macros with `#if configARMV9_SVE2` guards |
| `portmacro.h` | Type definitions, macros, config flag declarations |
| `README.md` | This file |
## Context Switch Frame Layout
### With `configARMV9_SVE2 = 0` (Armv8 compatible)
```
[SP] → FPU flag + critical nesting (16 bytes)
FPSR + FPCR (16 bytes)
Q0-Q31 (512 bytes)
SPSR + ELR (16 bytes)
X0-X30 (256 bytes)
```
### With `configARMV9_SVE2 = 1`
```
[SP] → FPU flag + critical nesting (16 bytes)
P0-P15 + FFR (3 × VL bytes, 16-byte aligned)
Z0-Z31 (32 × VL bytes)
FPSR + FPCR (16 bytes)
SPSR + ELR (16 bytes)
X0-X30 (256 bytes)
```
Total SVE frame at VL=128: 32×16 + 3×16 + 16 = 576 bytes
Total SVE frame at VL=256: 32×32 + 3×32 + 16 = 1136 bytes

126
portable/GCC/ARM_AARCH64_ARMV9/mte_port.c Normal file
View file

@ -0,0 +1,126 @@
/*
* FreeRTOS Kernel <DEVELOPMENT BRANCH>
* Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
* https://www.FreeRTOS.org
* https://github.com/FreeRTOS
*
*/
/*
* mte_port.c — MTE heap wrapper and stack tagging for FreeRTOS Armv9 port.
*
* Uses ARM ACLE intrinsics (arm_acle.h) for MISRA C:2012 compliance.
*
* configARMV9_MTE_HEAP: wraps pvPortMalloc/vPortFree with IRG/STG tagging
* configARMV9_MTE_STACK: tags task stack memory on creation
*/
#define PORTMEMORY_IMPLEMENTATION 1
#include "FreeRTOS.h"
#include "task.h"
#include <stdint.h>
#include <arm_acle.h>
#if ( configARMV9_MTE_HEAP == 1 )
void * pvPortMallocTagged( size_t xSize )
{
/* MISRA Ref 11.5.1 [Void pointer assignment] */
/* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-115 */
void * pv = pvPortMalloc( xSize );
if( pv == NULL )
{
return NULL;
}
/* Generate random tag (excludes tag 0 via GCR_EL1) */
pv = __arm_mte_create_random_tag( pv, 0 );
/* Tag all 16-byte granules in the allocation */
size_t xRounded = ( xSize + 15U ) & ~15U;
/* MISRA Ref 11.5.1 [Void pointer assignment] */
/* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-115 */
uint8_t * pucTagPtr = ( uint8_t * ) pv;
for( size_t i = 0; i < xRounded; i += 16U )
{
__arm_mte_set_tag( &pucTagPtr[ i ] );
}
return pv;
}
void vPortFreeTagged( void * pv )
{
if( pv == NULL )
{
return;
}
/* Strip tag (top byte) to recover the canonical address for the allocator */
/* MISRA Ref 11.6.1 [Pointer to integer conversion] */
/* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-116 */
uintptr_t xAddr = ( uintptr_t ) pv & 0x00FFFFFFFFFFFFFFULL;
void * pvUntagged = ( void * ) xAddr;
/* Re-tag freed region with tag 0 (use-after-free detection).
* Tag the first 64 bytes (covers most small allocations). */
/* MISRA Ref 11.5.1 [Void pointer assignment] */
/* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-115 */
uint8_t * pucPtr = ( uint8_t * ) pvUntagged;
for( size_t i = 0; i < 64U; i += 16U )
{
__arm_mte_set_tag( &pucPtr[ i ] );
}
vPortFree( pvUntagged );
}
#endif /* configARMV9_MTE_HEAP */
#if ( configARMV9_MTE_STACK == 1 )
void vPortMteTagStack( StackType_t * pxStack, uint32_t ulStackDepth )
{
/* MISRA Ref 11.5.1 [Void pointer assignment] */
/* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-115 */
void * pvAddr = ( void * ) pxStack;
size_t xBytes = ( size_t ) ulStackDepth * sizeof( StackType_t );
/* Generate a random tag for this task's stack */
pvAddr = __arm_mte_create_random_tag( pvAddr, 0 );
/* Tag all granules */
uint8_t * pucPtr = ( uint8_t * ) pvAddr;
for( size_t i = 0; i < xBytes; i += 16U )
{
__arm_mte_set_tag( &pucPtr[ i ] );
}
}
#endif /* configARMV9_MTE_STACK */

702
portable/GCC/ARM_AARCH64_ARMV9/port.c Normal file
View file

@ -0,0 +1,702 @@
/*
* FreeRTOS Kernel <DEVELOPMENT BRANCH>
* Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
* https://www.FreeRTOS.org
* https://github.com/FreeRTOS
*
*/
/* Standard includes. */
#include <stdlib.h>
#include <arm_acle.h>
/*
* MISRA C:2012 Deviations for this port file:
*
* MISRA Ref 4.3.1 [Inline assembly usage]
* Rationale: This is a hardware port — inline assembly is required
* to access AArch64 system registers (GIC, generic timer, PAC keys,
* MTE configuration) that have no C-language equivalent.
*
* MISRA Ref 11.1.1, 11.5.1, 11.6.1 [Pointer type conversions]
* Rationale: Pointer casts between void*, integer types, and function
* pointers are required for stack initialisation, MTE tag manipulation,
* and setting task entry points in the initial context frame.
*/
/* Scheduler includes. */
#include "FreeRTOS.h"
#include "task.h"
#ifndef configUNIQUE_INTERRUPT_PRIORITIES
#error "configUNIQUE_INTERRUPT_PRIORITIES must be defined. See www.FreeRTOS.org/Using-FreeRTOS-on-Cortex-A-Embedded-Processors.html"
#endif
#ifndef configSETUP_TICK_INTERRUPT
#error "configSETUP_TICK_INTERRUPT() must be defined. See www.FreeRTOS.org/Using-FreeRTOS-on-Cortex-A-Embedded-Processors.html"
#endif /* configSETUP_TICK_INTERRUPT */
#ifndef configMAX_API_CALL_INTERRUPT_PRIORITY
#error "configMAX_API_CALL_INTERRUPT_PRIORITY must be defined. See www.FreeRTOS.org/Using-FreeRTOS-on-Cortex-A-Embedded-Processors.html"
#endif
#if configMAX_API_CALL_INTERRUPT_PRIORITY == 0
#error "configMAX_API_CALL_INTERRUPT_PRIORITY must not be set to 0"
#endif
#if configMAX_API_CALL_INTERRUPT_PRIORITY > configUNIQUE_INTERRUPT_PRIORITIES
#error "configMAX_API_CALL_INTERRUPT_PRIORITY must be less than or equal to configUNIQUE_INTERRUPT_PRIORITIES as the lower the numeric priority value the higher the logical interrupt priority"
#endif
#if configUSE_PORT_OPTIMISED_TASK_SELECTION == 1
/* Check the configuration. */
#if ( configMAX_PRIORITIES > 32 )
#error "configUSE_PORT_OPTIMISED_TASK_SELECTION can only be set to 1 when configMAX_PRIORITIES is less than or equal to 32. It is very rare that a system requires more than 10 to 15 difference priorities as tasks that share a priority will time slice."
#endif
#endif /* configUSE_PORT_OPTIMISED_TASK_SELECTION */
/* In case security extensions are implemented. */
#if configMAX_API_CALL_INTERRUPT_PRIORITY <= ( configUNIQUE_INTERRUPT_PRIORITIES / 2 )
#error "configMAX_API_CALL_INTERRUPT_PRIORITY must be greater than ( configUNIQUE_INTERRUPT_PRIORITIES / 2 )"
#endif
/* Some vendor specific files default configCLEAR_TICK_INTERRUPT() in
* portmacro.h. */
#ifndef configCLEAR_TICK_INTERRUPT
#define configCLEAR_TICK_INTERRUPT()
#endif
/* A critical section is exited when the critical section nesting count reaches
* this value. */
#define portNO_CRITICAL_NESTING ( ( size_t ) 0 )
/* In all GICs 255 can be written to the priority mask register to unmask all
* (but the lowest) interrupt priority. */
#define portUNMASK_VALUE ( 0xFFUL )
/* Tasks are not created with a floating point context, but can be given a
* floating point context after they have been created. A variable is stored as
* part of the tasks context that holds portNO_FLOATING_POINT_CONTEXT if the task
* does not have an FPU context, or any other value if the task does have an FPU
* context. */
#define portNO_FLOATING_POINT_CONTEXT ( ( StackType_t ) 0 )
/* Constants required to setup the initial task context. */
#define portSP_ELx ( ( StackType_t ) 0x01 )
#define portSP_EL0 ( ( StackType_t ) 0x00 )
#if defined( GUEST )
#define portEL1 ( ( StackType_t ) 0x04 )
#define portINITIAL_PSTATE ( portEL1 | portSP_EL0 )
#else
#define portEL3 ( ( StackType_t ) 0x0c )
/* At the time of writing, the BSP only supports EL3. */
#define portINITIAL_PSTATE ( portEL3 | portSP_EL0 )
#endif
/* Masks all bits in the APSR other than the mode bits. */
#define portAPSR_MODE_BITS_MASK ( 0x0C )
/* The I bit in the DAIF bits. */
#define portDAIF_I ( 0x80 )
/* Macro to unmask all interrupt priorities. */
/* s3_0_c4_c6_0 is ICC_PMR_EL1. */
#define portCLEAR_INTERRUPT_MASK() \
{ \
__asm volatile ( "MSR DAIFSET, #2 \n" \
"DSB SY \n" \
"ISB SY \n" \
"MSR s3_0_c4_c6_0, %0 \n" \
"DSB SY \n" \
"ISB SY \n" \
"MSR DAIFCLR, #2 \n" \
"DSB SY \n" \
"ISB SY \n" \
::"r" ( portUNMASK_VALUE ) ); \
}
/* The space on the stack required to hold the FPU registers.
* There are 32 128-bit plus 2 64-bit status registers.*/
#define portFPU_REGISTER_WORDS ( (32 * 2) + 2 )
/*-----------------------------------------------------------*/
/*
* Starts the first task executing. This function is necessarily written in
* assembly code so is implemented in portASM.s.
*/
extern void vPortRestoreTaskContext( void );
/*
* If the application provides an implementation of vApplicationIRQHandler(),
* then it will get called directly without saving the FPU registers on
* interrupt entry, and this weak implementation of
* vApplicationFPUSafeIRQHandler() is just provided to remove linkage errors -
* it should never actually get called so its implementation contains a
* call to configASSERT() that will always fail.
*
* If the application provides its own implementation of
* vApplicationFPUSafeIRQHandler() then the implementation of
* vApplicationIRQHandler() provided in portASM.S will save the FPU registers
* before calling it.
*
* Therefore, if the application writer wants FPU registers to be saved on
* interrupt entry their IRQ handler must be called
* vApplicationFPUSafeIRQHandler(), and if the application writer does not want
* FPU registers to be saved on interrupt entry their IRQ handler must be
* called vApplicationIRQHandler().
*/
void vApplicationFPUSafeIRQHandler( uint32_t ulICCIAR ) __attribute__((weak) );
/*-----------------------------------------------------------*/
/* A variable is used to keep track of the critical section nesting. This
* variable has to be stored as part of the task context and must be initialised to
* a non zero value to ensure interrupts don't inadvertently become unmasked before
* the scheduler starts. As it is stored as part of the task context it will
* automatically be set to 0 when the first task is started. */
/* MISRA Ref 8.4.1 [Declaration shall be visible] */
/* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-84 */
volatile uint64_t ullCriticalNesting = 9999ULL;
/* Saved as part of the task context. If ullPortTaskHasFPUContext is non-zero
* then floating point context must be saved and restored for the task. */
/* MISRA Ref 8.4.1 [Declaration shall be visible] */
/* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-84 */
uint64_t ullPortTaskHasFPUContext = pdFALSE;
/* Set to 1 to pend a context switch from an ISR. */
/* MISRA Ref 8.4.1 [Declaration shall be visible] */
/* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-84 */
uint64_t ullPortYieldRequired = pdFALSE;
/* Counts the interrupt nesting depth. A context switch is only performed if
* if the nesting depth is 0. */
/* MISRA Ref 8.4.1 [Declaration shall be visible] */
/* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-84 */
uint64_t ullPortInterruptNesting = 0;
/* Used in the ASM code. */
/* MISRA Ref 8.4.1 [Declaration shall be visible] */
/* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-84 */
__attribute__( ( used ) ) const uint64_t ullMaxAPIPriorityMask = ( configMAX_API_CALL_INTERRUPT_PRIORITY << portPRIORITY_SHIFT );
/*-----------------------------------------------------------*/
/*
* See header file for description.
*/
StackType_t * pxPortInitialiseStack( StackType_t * pxTopOfStack,
TaskFunction_t pxCode,
void * pvParameters )
{
/* Setup the initial stack of the task. The stack is set exactly as
* expected by the portRESTORE_CONTEXT() macro. */
#if ( configARMV9_MTE_STACK == 1 )
/* Tag the entire stack with a random MTE tag using ACLE intrinsics.
* No pointer-to-integer casts — MISRA C:2012 compliant. */
{
void * pvTagged = __arm_mte_create_random_tag( ( void * ) pxTopOfStack, 0 );
size_t xStackBytes = ( size_t ) configMINIMAL_STACK_SIZE * sizeof( StackType_t );
uint8_t * pucPtr = ( uint8_t * ) pvTagged - xStackBytes;
for( size_t i = 0; i < xStackBytes; i += 16U )
{
__arm_mte_set_tag( &pucPtr[ i ] );
}
pxTopOfStack = ( StackType_t * ) pvTagged;
}
#endif /* configARMV9_MTE_STACK */
/* First all the general purpose registers. */
pxTopOfStack--;
*pxTopOfStack = 0x0101010101010101ULL; /* R1 */
pxTopOfStack--;
*pxTopOfStack = ( StackType_t ) pvParameters; /* R0 */
pxTopOfStack--;
*pxTopOfStack = 0x0303030303030303ULL; /* R3 */
pxTopOfStack--;
*pxTopOfStack = 0x0202020202020202ULL; /* R2 */
pxTopOfStack--;
*pxTopOfStack = 0x0505050505050505ULL; /* R5 */
pxTopOfStack--;
*pxTopOfStack = 0x0404040404040404ULL; /* R4 */
pxTopOfStack--;
*pxTopOfStack = 0x0707070707070707ULL; /* R7 */
pxTopOfStack--;
*pxTopOfStack = 0x0606060606060606ULL; /* R6 */
pxTopOfStack--;
*pxTopOfStack = 0x0909090909090909ULL; /* R9 */
pxTopOfStack--;
*pxTopOfStack = 0x0808080808080808ULL; /* R8 */
pxTopOfStack--;
*pxTopOfStack = 0x1111111111111111ULL; /* R11 */
pxTopOfStack--;
*pxTopOfStack = 0x1010101010101010ULL; /* R10 */
pxTopOfStack--;
*pxTopOfStack = 0x1313131313131313ULL; /* R13 */
pxTopOfStack--;
*pxTopOfStack = 0x1212121212121212ULL; /* R12 */
pxTopOfStack--;
*pxTopOfStack = 0x1515151515151515ULL; /* R15 */
pxTopOfStack--;
*pxTopOfStack = 0x1414141414141414ULL; /* R14 */
pxTopOfStack--;
*pxTopOfStack = 0x1717171717171717ULL; /* R17 */
pxTopOfStack--;
*pxTopOfStack = 0x1616161616161616ULL; /* R16 */
pxTopOfStack--;
*pxTopOfStack = 0x1919191919191919ULL; /* R19 */
pxTopOfStack--;
*pxTopOfStack = 0x1818181818181818ULL; /* R18 */
pxTopOfStack--;
*pxTopOfStack = 0x2121212121212121ULL; /* R21 */
pxTopOfStack--;
*pxTopOfStack = 0x2020202020202020ULL; /* R20 */
pxTopOfStack--;
*pxTopOfStack = 0x2323232323232323ULL; /* R23 */
pxTopOfStack--;
*pxTopOfStack = 0x2222222222222222ULL; /* R22 */
pxTopOfStack--;
*pxTopOfStack = 0x2525252525252525ULL; /* R25 */
pxTopOfStack--;
*pxTopOfStack = 0x2424242424242424ULL; /* R24 */
pxTopOfStack--;
*pxTopOfStack = 0x2727272727272727ULL; /* R27 */
pxTopOfStack--;
*pxTopOfStack = 0x2626262626262626ULL; /* R26 */
pxTopOfStack--;
*pxTopOfStack = 0x2929292929292929ULL; /* R29 */
pxTopOfStack--;
*pxTopOfStack = 0x2828282828282828ULL; /* R28 */
pxTopOfStack--;
*pxTopOfStack = ( StackType_t ) 0x00; /* XZR - has no effect, used so there are an even number of registers. */
pxTopOfStack--;
*pxTopOfStack = ( StackType_t ) 0x00; /* R30 - procedure call link register. */
pxTopOfStack--;
*pxTopOfStack = portINITIAL_PSTATE;
pxTopOfStack--;
*pxTopOfStack = ( StackType_t ) pxCode; /* Exception return address. */
#if ( configUSE_TASK_FPU_SUPPORT == 1 )
{
#if ( configARMV9_PAC == 1 )
/* Per-task PAC keys: generate unique keys at creation for isolation.
* Uses RNDR (FEAT_RNG) by default; falls back to a deterministic PRNG
* when configARMV9_PAC_DETERMINISTIC_KEYS is defined (for testing). */
{
uint64_t k0, k1, k2, k3, k4, k5, k6, k7;
#if ( ( defined( configARMV9_PAC_DETERMINISTIC_KEYS ) ) && ( configARMV9_PAC_DETERMINISTIC_KEYS == 1 ) )
{
/* Deterministic PRNG for reproducible test runs.
* xorshift64* seeded from the stack address. */
static uint64_t ullPacPrngState = 0xA5A5A5A5DEADBEEFULL;
#define PAC_PRNG_NEXT( s ) do { (s) ^= ((s) >> 12); (s) ^= ((s) << 25); (s) ^= ((s) >> 27); (s) *= 0x2545F4914F6CDD1DULL; } while(0)
PAC_PRNG_NEXT( ullPacPrngState ); k0 = ullPacPrngState;
PAC_PRNG_NEXT( ullPacPrngState ); k1 = ullPacPrngState;
PAC_PRNG_NEXT( ullPacPrngState ); k2 = ullPacPrngState;
PAC_PRNG_NEXT( ullPacPrngState ); k3 = ullPacPrngState;
PAC_PRNG_NEXT( ullPacPrngState ); k4 = ullPacPrngState;
PAC_PRNG_NEXT( ullPacPrngState ); k5 = ullPacPrngState;
PAC_PRNG_NEXT( ullPacPrngState ); k6 = ullPacPrngState;
PAC_PRNG_NEXT( ullPacPrngState ); k7 = ullPacPrngState;
#undef PAC_PRNG_NEXT
}
#else
{
/* Hardware RNG (FEAT_RNG): RNDR instruction. */
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k0));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k1));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k2));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k3));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k4));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k5));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k6));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k7));
}
#endif
/* Stack layout (reverse order — APDB first pushed, APIA last):
* APDB Hi, APDB Lo, APDA Hi, APDA Lo, APIB Hi, APIB Lo, APIA Hi, APIA Lo */
pxTopOfStack--; *pxTopOfStack = k7; /* APDB Hi */
pxTopOfStack--; *pxTopOfStack = k6; /* APDB Lo */
pxTopOfStack--; *pxTopOfStack = k5; /* APDA Hi */
pxTopOfStack--; *pxTopOfStack = k4; /* APDA Lo */
pxTopOfStack--; *pxTopOfStack = k3; /* APIB Hi */
pxTopOfStack--; *pxTopOfStack = k2; /* APIB Lo */
pxTopOfStack--; *pxTopOfStack = k1; /* APIA Hi */
pxTopOfStack--; *pxTopOfStack = k0; /* APIA Lo */
}
#endif /* configARMV9_PAC */
/* The task will start with a critical nesting count of 0 as interrupts are
* enabled. */
pxTopOfStack--;
*pxTopOfStack = portNO_CRITICAL_NESTING;
/* The task will start without a floating point context. A task that
* uses the floating point hardware must call vPortTaskUsesFPU() before
* executing any floating point instructions. */
pxTopOfStack--;
*pxTopOfStack = portNO_FLOATING_POINT_CONTEXT;
#if ( configARMV9_TASK_VL == 1 )
/* Initial ZCR_EL1: max VL (0xF = implementation maximum).
* LDP X9, XZR reads X9 from [SP] (low addr), XZR from [SP+8] (high addr).
* So ZCR value must be at the lower address. */
pxTopOfStack--;
*pxTopOfStack = 0; /* pad (XZR slot, higher address) */
pxTopOfStack--;
*pxTopOfStack = 0xF; /* ZCR_EL1 (X9 slot, lower address = SP) */
#endif
}
#elif ( configUSE_TASK_FPU_SUPPORT == 2 )
{
/* The task will start with a floating point context. Leave enough
* space for the registers - and ensure they are initialised to 0. */
pxTopOfStack -= portFPU_REGISTER_WORDS;
memset( pxTopOfStack, 0x00, portFPU_REGISTER_WORDS * sizeof( StackType_t ) );
/* The task will start with a critical nesting count of 0 as interrupts are
* enabled. */
pxTopOfStack--;
*pxTopOfStack = portNO_CRITICAL_NESTING;
pxTopOfStack--;
*pxTopOfStack = pdTRUE;
ullPortTaskHasFPUContext = pdTRUE;
}
#else /* if ( configUSE_TASK_FPU_SUPPORT == 1 ) */
{
#error "Invalid configUSE_TASK_FPU_SUPPORT setting - configUSE_TASK_FPU_SUPPORT must be set to 1, 2, or left undefined."
}
#endif /* if ( configUSE_TASK_FPU_SUPPORT == 1 ) */
return pxTopOfStack;
}
/*-----------------------------------------------------------*/
BaseType_t xPortStartScheduler( void )
{
uint64_t ulAPSR;
__asm volatile ( "MRS %0, CurrentEL" : "=r" ( ulAPSR ) );
ulAPSR &= portAPSR_MODE_BITS_MASK;
#if defined( GUEST )
configASSERT( ulAPSR == portEL1 );
if( ulAPSR == portEL1 )
#else
configASSERT( ulAPSR == portEL3 );
if( ulAPSR == portEL3 )
#endif
{
/* Interrupts are turned off in the CPU itself to ensure a tick does
* not execute while the scheduler is being started. Interrupts are
* automatically turned back on in the CPU when the first task starts
* executing. */
portDISABLE_INTERRUPTS();
/* Start the timer that generates the tick ISR. */
configSETUP_TICK_INTERRUPT();
/* Start the first task executing. */
vPortRestoreTaskContext();
}
return 0;
}
/*-----------------------------------------------------------*/
void vPortEndScheduler( void )
{
/* Not implemented in ports where there is nothing to return to.
* Artificially force an assert. */
configASSERT( ullCriticalNesting == 1000ULL );
}
/*-----------------------------------------------------------*/
void vPortEnterCritical( void )
{
/* Mask interrupts up to the max syscall interrupt priority. */
( void ) uxPortSetInterruptMask();
/* Now interrupts are disabled ullCriticalNesting can be accessed
* directly. Increment ullCriticalNesting to keep a count of how many times
* portENTER_CRITICAL() has been called. */
ullCriticalNesting++;
/* This is not the interrupt safe version of the enter critical function so
* assert() if it is being called from an interrupt context. Only API
* functions that end in "FromISR" can be used in an interrupt. Only assert if
* the critical nesting count is 1 to protect against recursive calls if the
* assert function also uses a critical section. */
if( ullCriticalNesting == 1ULL )
{
configASSERT( ullPortInterruptNesting == 0 );
}
}
/*-----------------------------------------------------------*/
void vPortExitCritical( void )
{
if( ullCriticalNesting > portNO_CRITICAL_NESTING )
{
/* Decrement the nesting count as the critical section is being
* exited. */
ullCriticalNesting--;
/* If the nesting level has reached zero then all interrupt
* priorities must be re-enabled. */
if( ullCriticalNesting == portNO_CRITICAL_NESTING )
{
/* Critical nesting has reached zero so all interrupt priorities
* should be unmasked. */
portCLEAR_INTERRUPT_MASK();
}
}
}
/*-----------------------------------------------------------*/
void FreeRTOS_Tick_Handler( void )
{
/* Must be the lowest possible priority. */
#if !defined( QEMU )
{
uint64_t ullRunningInterruptPriority;
/* s3_0_c12_c11_3 is ICC_RPR_EL1. */
__asm volatile ( "MRS %0, s3_0_c12_c11_3" : "=r" ( ullRunningInterruptPriority ) );
configASSERT( ullRunningInterruptPriority == ( portLOWEST_USABLE_INTERRUPT_PRIORITY << portPRIORITY_SHIFT ) );
( void ) ullRunningInterruptPriority;
}
#endif
/* Interrupts should not be enabled before this point. */
#if ( configASSERT_DEFINED == 1 )
{
uint64_t ulMaskBits;
__asm volatile ( "MRS %0, DAIF" : "=r" ( ulMaskBits )::"memory" );
configASSERT( ( ulMaskBits & portDAIF_I ) != 0 );
}
#endif /* configASSERT_DEFINED */
/* Set interrupt mask before altering scheduler structures. The tick
* handler runs at the lowest priority, so interrupts cannot already be masked,
* so there is no need to save and restore the current mask value. It is
* necessary to turn off interrupts in the CPU itself while the ICCPMR is being
* updated. */
/* s3_0_c4_c6_0 is ICC_PMR_EL1. */
__asm volatile ( "MSR s3_0_c4_c6_0, %0 \n"
"DSB SY \n"
"ISB SY \n"
::"r" ( ( uint64_t ) configMAX_API_CALL_INTERRUPT_PRIORITY << portPRIORITY_SHIFT ) : "memory" );
/* Ok to enable interrupts after the interrupt source has been cleared. */
configCLEAR_TICK_INTERRUPT();
/* NOTE: Do NOT call portENABLE_INTERRUPTS() here - we are inside the IRQ
* handler and re-enabling DAIF.I would allow nested timer interrupts that
* corrupt the handler's stack frame. The PMR-based priority masking is
* sufficient to allow higher-priority interrupts if needed. */
/* Increment the RTOS tick. */
if( xTaskIncrementTick() != pdFALSE )
{
ullPortYieldRequired = pdTRUE;
}
/* Restore interrupt priority mask (PMR) without enabling DAIF.I.
* The ERET at the end of the IRQ handler will restore PSTATE. */
__asm volatile ( "MSR s3_0_c4_c6_0, %0 \n"
"DSB SY \n"
"ISB SY \n"
::"r" ( ( uint64_t ) 0xff ) );
}
/*-----------------------------------------------------------*/
#if ( configUSE_TASK_FPU_SUPPORT != 2 )
void vPortTaskUsesFPU( void )
{
/* A task is registering the fact that it needs an FPU context. Set the
* FPU flag (which is saved as part of the task context). */
ullPortTaskHasFPUContext = pdTRUE;
/* Consider initialising the FPSR here - but probably not necessary in
* AArch64. */
}
#endif /* configUSE_TASK_FPU_SUPPORT */
/*-----------------------------------------------------------*/
void vPortClearInterruptMask( UBaseType_t uxNewMaskValue )
{
if( uxNewMaskValue == pdFALSE )
{
portCLEAR_INTERRUPT_MASK();
}
}
/*-----------------------------------------------------------*/
UBaseType_t uxPortSetInterruptMask( void )
{
uint32_t ulReturn;
uint64_t ullPMRValue;
/* Interrupt in the CPU must be turned off while the ICCPMR is being
* updated. */
portDISABLE_INTERRUPTS();
/* s3_0_c4_c6_0 is ICC_PMR_EL1. */
__asm volatile ( "MRS %0, s3_0_c4_c6_0" : "=r" ( ullPMRValue ) );
if( ullPMRValue == ( configMAX_API_CALL_INTERRUPT_PRIORITY << portPRIORITY_SHIFT ) )
{
/* Interrupts were already masked. */
ulReturn = pdTRUE;
}
else
{
ulReturn = pdFALSE;
/* s3_0_c4_c6_0 is ICC_PMR_EL1. */
__asm volatile ( "MSR s3_0_c4_c6_0, %0 \n"
"DSB SY \n"
"ISB SY \n"
::"r" ( ( uint64_t ) configMAX_API_CALL_INTERRUPT_PRIORITY << portPRIORITY_SHIFT ) : "memory" );
}
/* Re-enable DAIF.I now that PMR is set. PMR-based priority masking
* prevents interrupts below configMAX_API_CALL_INTERRUPT_PRIORITY
* from being taken. This matches the upstream ARM_AARCH64_SRE port. */
portENABLE_INTERRUPTS();
return ulReturn;
}
/*-----------------------------------------------------------*/
#if ( configASSERT_DEFINED == 1 )
void vPortValidateInterruptPriority( void )
{
/* The following assertion will fail if a service routine (ISR) for
* an interrupt that has been assigned a priority above
* configMAX_SYSCALL_INTERRUPT_PRIORITY calls an ISR safe FreeRTOS API
* function. ISR safe FreeRTOS API functions must *only* be called
* from interrupts that have been assigned a priority at or below
* configMAX_SYSCALL_INTERRUPT_PRIORITY.
*
* Numerically low interrupt priority numbers represent logically high
* interrupt priorities, therefore the priority of the interrupt must
* be set to a value equal to or numerically *higher* than
* configMAX_SYSCALL_INTERRUPT_PRIORITY.
*
* FreeRTOS maintains separate thread and ISR API functions to ensure
* interrupt entry is as fast and simple as possible. */
uint64_t ullRunningInterruptPriority;
/* s3_0_c12_c11_3 is ICC_RPR_EL1. */
__asm volatile ( "MRS %0, s3_0_c12_c11_3" : "=r" ( ullRunningInterruptPriority ) );
configASSERT( ullRunningInterruptPriority >= ( configMAX_API_CALL_INTERRUPT_PRIORITY << portPRIORITY_SHIFT ) );
}
#endif /* configASSERT_DEFINED */
/*-----------------------------------------------------------*/
void vApplicationFPUSafeIRQHandler( uint32_t ulICCIAR )
{
( void ) ulICCIAR;
configASSERT( ( volatile void * ) NULL );
}
/*-----------------------------------------------------------*/
#if ( configARMV9_PAC == 1 )
void vPortTaskRegeneratePACKeys( void )
{
/* Regenerate PAC keys for the current task using RNDR.
* Called from task context. Keys take effect immediately (MSR + ISB). */
uint64_t k0, k1, k2, k3, k4, k5, k6, k7;
#if ( ( defined( configARMV9_PAC_DETERMINISTIC_KEYS ) ) && ( configARMV9_PAC_DETERMINISTIC_KEYS == 1 ) )
{
static uint64_t ullRegenState = 0xDEADC0DEBEEF1234ULL;
#define PAC_REGEN_NEXT( s ) do { (s) ^= ((s) >> 12); (s) ^= ((s) << 25); (s) ^= ((s) >> 27); (s) *= 0x2545F4914F6CDD1DULL; } while(0)
PAC_REGEN_NEXT( ullRegenState ); k0 = ullRegenState;
PAC_REGEN_NEXT( ullRegenState ); k1 = ullRegenState;
PAC_REGEN_NEXT( ullRegenState ); k2 = ullRegenState;
PAC_REGEN_NEXT( ullRegenState ); k3 = ullRegenState;
PAC_REGEN_NEXT( ullRegenState ); k4 = ullRegenState;
PAC_REGEN_NEXT( ullRegenState ); k5 = ullRegenState;
PAC_REGEN_NEXT( ullRegenState ); k6 = ullRegenState;
PAC_REGEN_NEXT( ullRegenState ); k7 = ullRegenState;
#undef PAC_REGEN_NEXT
}
#else
{
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k0));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k1));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k2));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k3));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k4));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k5));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k6));
__asm volatile("mrs %0, s3_3_c2_c4_0" : "=r"(k7));
}
#endif
__asm volatile(
"msr APIAKeyLo_EL1, %0\n"
"msr APIAKeyHi_EL1, %1\n"
"msr APIBKeyLo_EL1, %2\n"
"msr APIBKeyHi_EL1, %3\n"
"msr APDAKeyLo_EL1, %4\n"
"msr APDAKeyHi_EL1, %5\n"
"msr APDBKeyLo_EL1, %6\n"
"msr APDBKeyHi_EL1, %7\n"
"isb\n"
:: "r"(k0), "r"(k1), "r"(k2), "r"(k3),
"r"(k4), "r"(k5), "r"(k6), "r"(k7)
);
}
#endif /* configARMV9_PAC */
#if ( configARMV9_TASK_VL == 1 )
void vPortTaskSetVL( uint32_t ulVL )
{
/* Set the SVE vector length for the current task.
* ulVL is the ZCR_EL1.LEN field value: VL = (LEN+1) * 128 bits.
* E.g., ulVL=0 → 128-bit, ulVL=1 → 256-bit, ulVL=3 → 512-bit.
* The actual VL is clamped by the implementation's maximum.
* ZCR_EL1 = S3_0_C1_C2_0 */
__asm volatile( "MSR S3_0_C1_C2_0, %0\n ISB\n" :: "r"( (uint64_t)ulVL ) );
}
#endif /* configARMV9_TASK_VL */

792
portable/GCC/ARM_AARCH64_ARMV9/portASM.S Normal file
View file

@ -0,0 +1,792 @@
/*
* FreeRTOS Kernel <DEVELOPMENT BRANCH>
* Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
* https://www.FreeRTOS.org
* https://github.com/FreeRTOS
*
*/
.text
/* Variables and functions. */
.extern ullMaxAPIPriorityMask
.extern pxCurrentTCB
.extern vTaskSwitchContext
.extern vApplicationIRQHandler
.extern ullPortInterruptNesting
.extern ullPortTaskHasFPUContext
.extern ullCriticalNesting
.extern ullPortYieldRequired
.extern _freertos_vector_table
.global FreeRTOS_IRQ_Handler
.global FreeRTOS_SWI_Handler
.global vPortRestoreTaskContext
.macro portSAVE_CONTEXT
/* Switch to use the EL0 stack pointer. */
MSR SPSEL, #0
/* Save the entire context. */
STP X0, X1, [SP, #-0x10]!
STP X2, X3, [SP, #-0x10]!
STP X4, X5, [SP, #-0x10]!
STP X6, X7, [SP, #-0x10]!
STP X8, X9, [SP, #-0x10]!
STP X10, X11, [SP, #-0x10]!
STP X12, X13, [SP, #-0x10]!
STP X14, X15, [SP, #-0x10]!
STP X16, X17, [SP, #-0x10]!
STP X18, X19, [SP, #-0x10]!
STP X20, X21, [SP, #-0x10]!
STP X22, X23, [SP, #-0x10]!
STP X24, X25, [SP, #-0x10]!
STP X26, X27, [SP, #-0x10]!
STP X28, X29, [SP, #-0x10]!
#if ( configARMV9_PAC_FRAME == 1 )
PACIA X30, SP /* Sign LR with SP as modifier before saving */
#endif
STP X30, XZR, [SP, #-0x10]!
/* Save the SPSR. */
#if defined( GUEST )
MRS X3, SPSR_EL1
MRS X2, ELR_EL1
#else
MRS X3, SPSR_EL3
/* Save the ELR. */
MRS X2, ELR_EL3
#endif
STP X2, X3, [SP, #-0x10]!
/* Save the critical section nesting depth. */
LDR X0, ullCriticalNestingConst
LDR X3, [X0]
/* Save the FPU context indicator. */
LDR X0, ullPortTaskHasFPUContextConst
LDR X2, [X0]
/* Save the FPU/SVE context, if any. */
CMP X2, #0
B.EQ 1f
#if ( configARMV9_SVE2 == 1 )
/* SVE2: save FPSR/FPCR first (SP aligned), then Z0-Z31, then P0-P15+FFR */
MRS X9, FPSR
MRS X10, FPCR
STP X9, X10, [SP, #-0x10]!
ADDVL SP, SP, #-32
STR Z0, [SP, #0, MUL VL]
STR Z1, [SP, #1, MUL VL]
STR Z2, [SP, #2, MUL VL]
STR Z3, [SP, #3, MUL VL]
STR Z4, [SP, #4, MUL VL]
STR Z5, [SP, #5, MUL VL]
STR Z6, [SP, #6, MUL VL]
STR Z7, [SP, #7, MUL VL]
STR Z8, [SP, #8, MUL VL]
STR Z9, [SP, #9, MUL VL]
STR Z10, [SP, #10, MUL VL]
STR Z11, [SP, #11, MUL VL]
STR Z12, [SP, #12, MUL VL]
STR Z13, [SP, #13, MUL VL]
STR Z14, [SP, #14, MUL VL]
STR Z15, [SP, #15, MUL VL]
STR Z16, [SP, #16, MUL VL]
STR Z17, [SP, #17, MUL VL]
STR Z18, [SP, #18, MUL VL]
STR Z19, [SP, #19, MUL VL]
STR Z20, [SP, #20, MUL VL]
STR Z21, [SP, #21, MUL VL]
STR Z22, [SP, #22, MUL VL]
STR Z23, [SP, #23, MUL VL]
STR Z24, [SP, #24, MUL VL]
STR Z25, [SP, #25, MUL VL]
STR Z26, [SP, #26, MUL VL]
STR Z27, [SP, #27, MUL VL]
STR Z28, [SP, #28, MUL VL]
STR Z29, [SP, #29, MUL VL]
STR Z30, [SP, #30, MUL VL]
STR Z31, [SP, #31, MUL VL]
ADDVL SP, SP, #-3
STR P0, [SP, #0, MUL VL]
STR P1, [SP, #1, MUL VL]
STR P2, [SP, #2, MUL VL]
STR P3, [SP, #3, MUL VL]
STR P4, [SP, #4, MUL VL]
STR P5, [SP, #5, MUL VL]
STR P6, [SP, #6, MUL VL]
STR P7, [SP, #7, MUL VL]
STR P8, [SP, #8, MUL VL]
STR P9, [SP, #9, MUL VL]
STR P10, [SP, #10, MUL VL]
STR P11, [SP, #11, MUL VL]
STR P12, [SP, #12, MUL VL]
STR P13, [SP, #13, MUL VL]
STR P14, [SP, #14, MUL VL]
STR P15, [SP, #15, MUL VL]
RDFFR P0.B
STR P0, [SP, #16, MUL VL]
#else /* configARMV9_SVE2 */
STP Q0, Q1, [SP,#-0x20]!
STP Q2, Q3, [SP,#-0x20]!
STP Q4, Q5, [SP,#-0x20]!
STP Q6, Q7, [SP,#-0x20]!
STP Q8, Q9, [SP,#-0x20]!
STP Q10, Q11, [SP,#-0x20]!
STP Q12, Q13, [SP,#-0x20]!
STP Q14, Q15, [SP,#-0x20]!
STP Q16, Q17, [SP,#-0x20]!
STP Q18, Q19, [SP,#-0x20]!
STP Q20, Q21, [SP,#-0x20]!
STP Q22, Q23, [SP,#-0x20]!
STP Q24, Q25, [SP,#-0x20]!
STP Q26, Q27, [SP,#-0x20]!
STP Q28, Q29, [SP,#-0x20]!
STP Q30, Q31, [SP,#-0x20]!
MRS X9, FPSR
MRS X10, FPCR
STP X9, X10, [SP, #-0x10]!
#endif /* configARMV9_SVE2 */
1:
#if ( configARMV9_PAC == 1 )
/* Save PAC keys. configARMV9_PAC_KEYS controls how many key pairs:
* 4 = all (APIA+APIB+APDA+APDB), 2 = APIA+APDA, 1 = APIA only. */
MRS X9, APIAKeyLo_EL1
MRS X10, APIAKeyHi_EL1
STP X9, X10, [SP, #-0x10]!
#if ( !defined( configARMV9_PAC_KEYS ) || configARMV9_PAC_KEYS >= 4 )
MRS X9, APIBKeyLo_EL1
MRS X10, APIBKeyHi_EL1
STP X9, X10, [SP, #-0x10]!
#endif
#if ( !defined( configARMV9_PAC_KEYS ) || configARMV9_PAC_KEYS >= 2 )
MRS X9, APDAKeyLo_EL1
MRS X10, APDAKeyHi_EL1
STP X9, X10, [SP, #-0x10]!
#endif
#if ( !defined( configARMV9_PAC_KEYS ) || configARMV9_PAC_KEYS >= 4 )
MRS X9, APDBKeyLo_EL1
MRS X10, APDBKeyHi_EL1
STP X9, X10, [SP, #-0x10]!
#endif
#endif /* configARMV9_PAC */
/* Store the critical nesting count and FPU context indicator. */
STP X2, X3, [SP, #-0x10]!
LDR X0, pxCurrentTCBConst
LDR X1, [X0]
MOV X0, SP /* Move SP into X0 for saving. */
STR X0, [X1]
/* Switch to use the ELx stack pointer. */
MSR SPSEL, #1
.endm
; /**********************************************************************/
.macro portRESTORE_CONTEXT
/* Switch to use the EL0 stack pointer. */
MSR SPSEL, #0
/* Set the SP to point to the stack of the task being restored. */
LDR X0, pxCurrentTCBConst
LDR X1, [X0]
LDR X0, [X1]
MOV SP, X0
LDP X2, X3, [SP], #0x10 /* Critical nesting and FPU context. */
#if ( configARMV9_PAC == 1 )
/* Restore PAC keys (reverse order of save). */
#if ( !defined( configARMV9_PAC_KEYS ) || configARMV9_PAC_KEYS >= 4 )
LDP X9, X10, [SP], #0x10
MSR APDBKeyLo_EL1, X9
MSR APDBKeyHi_EL1, X10
#endif
#if ( !defined( configARMV9_PAC_KEYS ) || configARMV9_PAC_KEYS >= 2 )
LDP X9, X10, [SP], #0x10
MSR APDAKeyLo_EL1, X9
MSR APDAKeyHi_EL1, X10
#endif
#if ( !defined( configARMV9_PAC_KEYS ) || configARMV9_PAC_KEYS >= 4 )
LDP X9, X10, [SP], #0x10
MSR APIBKeyLo_EL1, X9
MSR APIBKeyHi_EL1, X10
#endif
LDP X9, X10, [SP], #0x10
MSR APIAKeyLo_EL1, X9
MSR APIAKeyHi_EL1, X10
/* ISB deferred: merged with ISB after ICC_PMR_EL1 write below. */
#endif /* configARMV9_PAC */
/* Set the PMR register to be correct for the current critical nesting
depth. */
LDR X0, ullCriticalNestingConst /* X0 holds the address of ullCriticalNesting. */
MOV X1, #255 /* X1 holds the unmask value. */
CMP X3, #0
B.EQ 1f
LDR X6, ullMaxAPIPriorityMaskConst
LDR X1, [X6] /* X1 holds the mask value. */
1:
MSR s3_0_c4_c6_0, X1 /* Write the mask value to ICCPMR. s3_0_c4_c6_0 is ICC_PMR_EL1. */
DSB SY /* Required when ICC_CTLR_EL1.PMHE==1 (GIC IHI 0069H §12.1.6). */
ISB SY /* Synchronizes both PAC key and PMR writes. */
STR X3, [X0] /* Restore the task's critical nesting count. */
/* Restore the FPU context indicator. */
LDR X0, ullPortTaskHasFPUContextConst
STR X2, [X0]
/* Restore the FPU/SVE context, if any. */
CMP X2, #0
B.EQ 1f
#if ( configARMV9_SVE2 == 1 )
/* SVE2: restore P0-P15+FFR, then Z0-Z31, then FPSR/FPCR */
LDR P0, [SP, #16, MUL VL]
WRFFR P0.B
LDR P15, [SP, #15, MUL VL]
LDR P14, [SP, #14, MUL VL]
LDR P13, [SP, #13, MUL VL]
LDR P12, [SP, #12, MUL VL]
LDR P11, [SP, #11, MUL VL]
LDR P10, [SP, #10, MUL VL]
LDR P9, [SP, #9, MUL VL]
LDR P8, [SP, #8, MUL VL]
LDR P7, [SP, #7, MUL VL]
LDR P6, [SP, #6, MUL VL]
LDR P5, [SP, #5, MUL VL]
LDR P4, [SP, #4, MUL VL]
LDR P3, [SP, #3, MUL VL]
LDR P2, [SP, #2, MUL VL]
LDR P1, [SP, #1, MUL VL]
LDR P0, [SP, #0, MUL VL]
ADDVL SP, SP, #3
LDR Z31, [SP, #31, MUL VL]
LDR Z30, [SP, #30, MUL VL]
LDR Z29, [SP, #29, MUL VL]
LDR Z28, [SP, #28, MUL VL]
LDR Z27, [SP, #27, MUL VL]
LDR Z26, [SP, #26, MUL VL]
LDR Z25, [SP, #25, MUL VL]
LDR Z24, [SP, #24, MUL VL]
LDR Z23, [SP, #23, MUL VL]
LDR Z22, [SP, #22, MUL VL]
LDR Z21, [SP, #21, MUL VL]
LDR Z20, [SP, #20, MUL VL]
LDR Z19, [SP, #19, MUL VL]
LDR Z18, [SP, #18, MUL VL]
LDR Z17, [SP, #17, MUL VL]
LDR Z16, [SP, #16, MUL VL]
LDR Z15, [SP, #15, MUL VL]
LDR Z14, [SP, #14, MUL VL]
LDR Z13, [SP, #13, MUL VL]
LDR Z12, [SP, #12, MUL VL]
LDR Z11, [SP, #11, MUL VL]
LDR Z10, [SP, #10, MUL VL]
LDR Z9, [SP, #9, MUL VL]
LDR Z8, [SP, #8, MUL VL]
LDR Z7, [SP, #7, MUL VL]
LDR Z6, [SP, #6, MUL VL]
LDR Z5, [SP, #5, MUL VL]
LDR Z4, [SP, #4, MUL VL]
LDR Z3, [SP, #3, MUL VL]
LDR Z2, [SP, #2, MUL VL]
LDR Z1, [SP, #1, MUL VL]
LDR Z0, [SP, #0, MUL VL]
ADDVL SP, SP, #31
ADDVL SP, SP, #1
LDP X9, X10, [SP], #0x10
MSR FPSR, X9
MSR FPCR, X10
#else /* configARMV9_SVE2 */
LDP X9, X10, [SP], #0x10
LDP Q30, Q31, [SP], #0x20
LDP Q28, Q29, [SP], #0x20
LDP Q26, Q27, [SP], #0x20
LDP Q24, Q25, [SP], #0x20
LDP Q22, Q23, [SP], #0x20
LDP Q20, Q21, [SP], #0x20
LDP Q18, Q19, [SP], #0x20
LDP Q16, Q17, [SP], #0x20
LDP Q14, Q15, [SP], #0x20
LDP Q12, Q13, [SP], #0x20
LDP Q10, Q11, [SP], #0x20
LDP Q8, Q9, [SP], #0x20
LDP Q6, Q7, [SP], #0x20
LDP Q4, Q5, [SP], #0x20
LDP Q2, Q3, [SP], #0x20
LDP Q0, Q1, [SP], #0x20
MSR FPSR, X9
MSR FPCR, X10
#endif /* configARMV9_SVE2 */
1:
LDP X2, X3, [SP], #0x10 /* SPSR and ELR. */
#if defined( GUEST )
/* Restore the SPSR. */
MSR SPSR_EL1, X3
/* Restore the ELR. */
MSR ELR_EL1, X2
#else
/* Restore the SPSR. */
MSR SPSR_EL3, X3 /*_RB_ Assumes started in EL3. */
/* Restore the ELR. */
MSR ELR_EL3, X2
#endif
LDP X30, XZR, [SP], #0x10
#if ( configARMV9_PAC_FRAME == 1 )
AUTIA X30, SP /* Verify LR signature — faults if frame was corrupted */
#endif
LDP X28, X29, [SP], #0x10
LDP X26, X27, [SP], #0x10
LDP X24, X25, [SP], #0x10
LDP X22, X23, [SP], #0x10
LDP X20, X21, [SP], #0x10
LDP X18, X19, [SP], #0x10
LDP X16, X17, [SP], #0x10
LDP X14, X15, [SP], #0x10
LDP X12, X13, [SP], #0x10
LDP X10, X11, [SP], #0x10
LDP X8, X9, [SP], #0x10
LDP X6, X7, [SP], #0x10
LDP X4, X5, [SP], #0x10
LDP X2, X3, [SP], #0x10
LDP X0, X1, [SP], #0x10
/* Switch to use the ELx stack pointer. _RB_ Might not be required. */
MSR SPSEL, #1
ERET
.endm
/******************************************************************************
* FreeRTOS_SWI_Handler handler is used to perform a context switch.
*****************************************************************************/
.align 8
.type FreeRTOS_SWI_Handler, %function
FreeRTOS_SWI_Handler:
bti j
/* Save the context of the current task and select a new task to run. */
portSAVE_CONTEXT
#if defined( GUEST )
MRS X0, ESR_EL1
#else
MRS X0, ESR_EL3
#endif
LSR X1, X0, #26
#if defined( GUEST )
CMP X1, #0x15 /* 0x15 = SVC instruction. */
#else
CMP X1, #0x17 /* 0x17 = SMC instruction. */
#endif
B.NE FreeRTOS_Abort
BL vTaskSwitchContext
portRESTORE_CONTEXT
FreeRTOS_Abort:
/* Full ESR is in X0, exception class code is in X1. */
B .
/******************************************************************************
* vPortRestoreTaskContext is used to start the scheduler.
*****************************************************************************/
.align 8
.type vPortRestoreTaskContext, %function
vPortRestoreTaskContext:
bti c
.set freertos_vector_base, _freertos_vector_table
/* Install the FreeRTOS interrupt handlers. */
LDR X1, =freertos_vector_base
#if defined( GUEST )
MSR VBAR_EL1, X1
#else
MSR VBAR_EL3, X1
#endif
DSB SY
ISB SY
/* Start the first task. */
portRESTORE_CONTEXT
/******************************************************************************
* FreeRTOS_IRQ_Handler handles IRQ entry and exit.
* This handler is supposed to be used only for IRQs and never for FIQs. Per ARM
* GIC documentation [1], Group 0 interrupts are always signaled as FIQs. Since
* this handler is only for IRQs, We can safely assume Group 1 while accessing
* Interrupt Acknowledge and End Of Interrupt registers and therefore, use
* ICC_IAR1_EL1 and ICC_EOIR1_EL1.
*
* [1] https://developer.arm.com/documentation/198123/0300/Arm-CoreLink-GIC-fundamentals
*****************************************************************************/
.align 8
.type FreeRTOS_IRQ_Handler, %function
FreeRTOS_IRQ_Handler:
bti j
/* Save volatile registers. */
STP X0, X1, [SP, #-0x10]!
STP X2, X3, [SP, #-0x10]!
STP X4, X5, [SP, #-0x10]!
STP X6, X7, [SP, #-0x10]!
STP X8, X9, [SP, #-0x10]!
STP X10, X11, [SP, #-0x10]!
STP X12, X13, [SP, #-0x10]!
STP X14, X15, [SP, #-0x10]!
STP X16, X17, [SP, #-0x10]!
STP X18, XZR, [SP, #-0x10]! /* X19 is callee-saved; preserved by BL vApplicationIRQHandler. */
STP X29, X30, [SP, #-0x10]!
/* Save the SPSR and ELR. */
#if defined( GUEST )
MRS X3, SPSR_EL1
MRS X2, ELR_EL1
#else
MRS X3, SPSR_EL3
MRS X2, ELR_EL3
#endif
STP X2, X3, [SP, #-0x10]!
/* Increment the interrupt nesting counter. */
LDR X5, ullPortInterruptNestingConst
LDR X1, [X5] /* Old nesting count in X1. */
ADD X6, X1, #1
STR X6, [X5] /* Address of nesting count variable in X5. */
/* Maintain the interrupt nesting information across the function call. */
STP X1, X5, [SP, #-0x10]!
/* Read interrupt ID from the interrupt acknowledge register and store it
in X0 for future parameter and interrupt clearing use. */
MRS X0, S3_0_C12_C12_0 /* S3_0_C12_C12_0 is ICC_IAR1_EL1. */
/* Maintain the interrupt ID value across the function call. */
STP X0, X1, [SP, #-0x10]!
/* Call the C handler. */
BL vApplicationIRQHandler
/* Disable interrupts. */
MSR DAIFSET, #2
ISB SY
/* Restore the interrupt ID value. */
LDP X0, X1, [SP], #0x10
/* End IRQ processing by writing interrupt ID value to the EOI register. */
MSR S3_0_C12_C12_1, X0 /* S3_0_C12_C12_1 is ICC_EOIR1_EL1. */
/* Restore the critical nesting count. */
LDP X1, X5, [SP], #0x10
STR X1, [X5]
/* Has interrupt nesting unwound? */
CMP X1, #0
B.NE Exit_IRQ_No_Context_Switch
/* Is a context switch required? */
LDR X0, ullPortYieldRequiredConst
LDR X1, [X0]
CMP X1, #0
B.EQ Exit_IRQ_No_Context_Switch
/* Reset ullPortYieldRequired to 0. */
MOV X2, #0
STR X2, [X0]
/* Restore volatile registers. */
LDP X4, X5, [SP], #0x10 /* SPSR and ELR. */
#if defined( GUEST )
MSR SPSR_EL1, X5
MSR ELR_EL1, X4
#else
MSR SPSR_EL3, X5 /*_RB_ Assumes started in EL3. */
MSR ELR_EL3, X4
#endif
/* DSB+ISB removed: portSAVE_CONTEXT follows, then ERET in portRESTORE_CONTEXT. */
LDP X29, X30, [SP], #0x10
LDP X18, XZR, [SP], #0x10
LDP X16, X17, [SP], #0x10
LDP X14, X15, [SP], #0x10
LDP X12, X13, [SP], #0x10
LDP X10, X11, [SP], #0x10
LDP X8, X9, [SP], #0x10
LDP X6, X7, [SP], #0x10
LDP X4, X5, [SP], #0x10
LDP X2, X3, [SP], #0x10
LDP X0, X1, [SP], #0x10
/* Save the context of the current task and select a new task to run. */
portSAVE_CONTEXT
BL vTaskSwitchContext
portRESTORE_CONTEXT
Exit_IRQ_No_Context_Switch:
/* Restore volatile registers. */
LDP X4, X5, [SP], #0x10 /* SPSR and ELR. */
#if defined( GUEST )
MSR SPSR_EL1, X5
MSR ELR_EL1, X4
#else
MSR SPSR_EL3, X5 /*_RB_ Assumes started in EL3. */
MSR ELR_EL3, X4
#endif
/* DSB+ISB removed: ERET is a context synchronization event (ARM ARM §D1.10.2). */
LDP X29, X30, [SP], #0x10
LDP X18, XZR, [SP], #0x10
LDP X16, X17, [SP], #0x10
LDP X14, X15, [SP], #0x10
LDP X12, X13, [SP], #0x10
LDP X10, X11, [SP], #0x10
LDP X8, X9, [SP], #0x10
LDP X6, X7, [SP], #0x10
LDP X4, X5, [SP], #0x10
LDP X2, X3, [SP], #0x10
LDP X0, X1, [SP], #0x10
ERET
/******************************************************************************
* If the application provides an implementation of vApplicationIRQHandler(),
* then it will get called directly without saving the FPU registers on
* interrupt entry, and this weak implementation of
* vApplicationIRQHandler() will not get called.
*
* If the application provides its own implementation of
* vApplicationFPUSafeIRQHandler() then this implementation of
* vApplicationIRQHandler() will be called, save the FPU registers, and then
* call vApplicationFPUSafeIRQHandler().
*
* Therefore, if the application writer wants FPU registers to be saved on
* interrupt entry their IRQ handler must be called
* vApplicationFPUSafeIRQHandler(), and if the application writer does not want
* FPU registers to be saved on interrupt entry their IRQ handler must be
* called vApplicationIRQHandler().
*****************************************************************************/
.align 8
.weak vApplicationIRQHandler
.type vApplicationIRQHandler, %function
vApplicationIRQHandler:
bti c
/* Save LR and FP on the stack */
STP X29, X30, [SP, #-0x10]!
/* Save FPU/SVE volatile registers around C handler call */
#if ( configARMV9_SVE2 == 1 )
MRS X9, FPSR
MRS X10, FPCR
STP X9, X10, [SP, #-0x10]!
ADDVL SP, SP, #-32
STR Z0, [SP, #0, MUL VL]
STR Z1, [SP, #1, MUL VL]
STR Z2, [SP, #2, MUL VL]
STR Z3, [SP, #3, MUL VL]
STR Z4, [SP, #4, MUL VL]
STR Z5, [SP, #5, MUL VL]
STR Z6, [SP, #6, MUL VL]
STR Z7, [SP, #7, MUL VL]
STR Z8, [SP, #8, MUL VL]
STR Z9, [SP, #9, MUL VL]
STR Z10, [SP, #10, MUL VL]
STR Z11, [SP, #11, MUL VL]
STR Z12, [SP, #12, MUL VL]
STR Z13, [SP, #13, MUL VL]
STR Z14, [SP, #14, MUL VL]
STR Z15, [SP, #15, MUL VL]
STR Z16, [SP, #16, MUL VL]
STR Z17, [SP, #17, MUL VL]
STR Z18, [SP, #18, MUL VL]
STR Z19, [SP, #19, MUL VL]
STR Z20, [SP, #20, MUL VL]
STR Z21, [SP, #21, MUL VL]
STR Z22, [SP, #22, MUL VL]
STR Z23, [SP, #23, MUL VL]
STR Z24, [SP, #24, MUL VL]
STR Z25, [SP, #25, MUL VL]
STR Z26, [SP, #26, MUL VL]
STR Z27, [SP, #27, MUL VL]
STR Z28, [SP, #28, MUL VL]
STR Z29, [SP, #29, MUL VL]
STR Z30, [SP, #30, MUL VL]
STR Z31, [SP, #31, MUL VL]
ADDVL SP, SP, #-3
STR P0, [SP, #0, MUL VL]
STR P1, [SP, #1, MUL VL]
STR P2, [SP, #2, MUL VL]
STR P3, [SP, #3, MUL VL]
STR P4, [SP, #4, MUL VL]
STR P5, [SP, #5, MUL VL]
STR P6, [SP, #6, MUL VL]
STR P7, [SP, #7, MUL VL]
STR P8, [SP, #8, MUL VL]
STR P9, [SP, #9, MUL VL]
STR P10, [SP, #10, MUL VL]
STR P11, [SP, #11, MUL VL]
STR P12, [SP, #12, MUL VL]
STR P13, [SP, #13, MUL VL]
STR P14, [SP, #14, MUL VL]
STR P15, [SP, #15, MUL VL]
#else
STP Q0, Q1, [SP,#-0x20]!
STP Q2, Q3, [SP,#-0x20]!
STP Q4, Q5, [SP,#-0x20]!
STP Q6, Q7, [SP,#-0x20]!
STP Q8, Q9, [SP,#-0x20]!
STP Q10, Q11, [SP,#-0x20]!
STP Q12, Q13, [SP,#-0x20]!
STP Q14, Q15, [SP,#-0x20]!
STP Q16, Q17, [SP,#-0x20]!
STP Q18, Q19, [SP,#-0x20]!
STP Q20, Q21, [SP,#-0x20]!
STP Q22, Q23, [SP,#-0x20]!
STP Q24, Q25, [SP,#-0x20]!
STP Q26, Q27, [SP,#-0x20]!
STP Q28, Q29, [SP,#-0x20]!
STP Q30, Q31, [SP,#-0x20]!
MRS X9, FPSR
MRS X10, FPCR
STP X9, X10, [SP, #-0x10]!
#endif /* configARMV9_SVE2 */
/* Call the C handler. */
BL vApplicationFPUSafeIRQHandler
/* Restore FPU/SVE volatile registers */
#if ( configARMV9_SVE2 == 1 )
LDR P15, [SP, #15, MUL VL]
LDR P14, [SP, #14, MUL VL]
LDR P13, [SP, #13, MUL VL]
LDR P12, [SP, #12, MUL VL]
LDR P11, [SP, #11, MUL VL]
LDR P10, [SP, #10, MUL VL]
LDR P9, [SP, #9, MUL VL]
LDR P8, [SP, #8, MUL VL]
LDR P7, [SP, #7, MUL VL]
LDR P6, [SP, #6, MUL VL]
LDR P5, [SP, #5, MUL VL]
LDR P4, [SP, #4, MUL VL]
LDR P3, [SP, #3, MUL VL]
LDR P2, [SP, #2, MUL VL]
LDR P1, [SP, #1, MUL VL]
LDR P0, [SP, #0, MUL VL]
ADDVL SP, SP, #3
LDR Z31, [SP, #31, MUL VL]
LDR Z30, [SP, #30, MUL VL]
LDR Z29, [SP, #29, MUL VL]
LDR Z28, [SP, #28, MUL VL]
LDR Z27, [SP, #27, MUL VL]
LDR Z26, [SP, #26, MUL VL]
LDR Z25, [SP, #25, MUL VL]
LDR Z24, [SP, #24, MUL VL]
LDR Z23, [SP, #23, MUL VL]
LDR Z22, [SP, #22, MUL VL]
LDR Z21, [SP, #21, MUL VL]
LDR Z20, [SP, #20, MUL VL]
LDR Z19, [SP, #19, MUL VL]
LDR Z18, [SP, #18, MUL VL]
LDR Z17, [SP, #17, MUL VL]
LDR Z16, [SP, #16, MUL VL]
LDR Z15, [SP, #15, MUL VL]
LDR Z14, [SP, #14, MUL VL]
LDR Z13, [SP, #13, MUL VL]
LDR Z12, [SP, #12, MUL VL]
LDR Z11, [SP, #11, MUL VL]
LDR Z10, [SP, #10, MUL VL]
LDR Z9, [SP, #9, MUL VL]
LDR Z8, [SP, #8, MUL VL]
LDR Z7, [SP, #7, MUL VL]
LDR Z6, [SP, #6, MUL VL]
LDR Z5, [SP, #5, MUL VL]
LDR Z4, [SP, #4, MUL VL]
LDR Z3, [SP, #3, MUL VL]
LDR Z2, [SP, #2, MUL VL]
LDR Z1, [SP, #1, MUL VL]
LDR Z0, [SP, #0, MUL VL]
ADDVL SP, SP, #31
ADDVL SP, SP, #1
LDP X9, X10, [SP], #0x10
MSR FPSR, X9
MSR FPCR, X10
#else
LDP X9, X10, [SP], #0x10
LDP Q30, Q31, [SP], #0x20
LDP Q28, Q29, [SP], #0x20
LDP Q26, Q27, [SP], #0x20
LDP Q24, Q25, [SP], #0x20
LDP Q22, Q23, [SP], #0x20
LDP Q20, Q21, [SP], #0x20
LDP Q18, Q19, [SP], #0x20
LDP Q16, Q17, [SP], #0x20
LDP Q14, Q15, [SP], #0x20
LDP Q12, Q13, [SP], #0x20
LDP Q10, Q11, [SP], #0x20
LDP Q8, Q9, [SP], #0x20
LDP Q6, Q7, [SP], #0x20
LDP Q4, Q5, [SP], #0x20
LDP Q2, Q3, [SP], #0x20
LDP Q0, Q1, [SP], #0x20
MSR FPSR, X9
MSR FPCR, X10
#endif /* configARMV9_SVE2 */
/* Restore FP and LR */
LDP X29, X30, [SP], #0x10
RET
.align 8
pxCurrentTCBConst: .dword pxCurrentTCB
ullCriticalNestingConst: .dword ullCriticalNesting
ullPortTaskHasFPUContextConst: .dword ullPortTaskHasFPUContext
ullMaxAPIPriorityMaskConst: .dword ullMaxAPIPriorityMask
ullPortInterruptNestingConst: .dword ullPortInterruptNesting
ullPortYieldRequiredConst: .dword ullPortYieldRequired
.end

259
portable/GCC/ARM_AARCH64_ARMV9/portmacro.h Normal file
View file

@ -0,0 +1,259 @@
/*
* FreeRTOS Kernel <DEVELOPMENT BRANCH>
* Copyright (C) 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
* https://www.FreeRTOS.org
* https://github.com/FreeRTOS
*
*/
#ifndef PORTMACRO_H
#define PORTMACRO_H
/* *INDENT-OFF* */
#ifdef __cplusplus
extern "C" {
#endif
/* *INDENT-ON* */
/*-----------------------------------------------------------
* Port specific definitions.
*
* The settings in this file configure FreeRTOS correctly for the given hardware
* and compiler.
*
* These settings should not be altered.
*-----------------------------------------------------------
*/
/* Type definitions. */
#define portCHAR char
#define portFLOAT float
#define portDOUBLE double
#define portLONG long
#define portSHORT short
#define portSTACK_TYPE size_t
#define portBASE_TYPE long
typedef portSTACK_TYPE StackType_t;
typedef portBASE_TYPE BaseType_t;
typedef uint64_t UBaseType_t;
typedef uint64_t TickType_t;
#define portMAX_DELAY ( ( TickType_t ) 0xffffffffffffffffULL )
/* 32-bit tick type on a 32-bit architecture, so reads of the tick count do
* not need to be guarded with a critical section. */
#define portTICK_TYPE_IS_ATOMIC 1
/*-----------------------------------------------------------*/
/* Hardware specifics. */
#define portSTACK_GROWTH ( -1 )
#define portTICK_PERIOD_MS ( ( TickType_t ) 1000 / configTICK_RATE_HZ )
#define portBYTE_ALIGNMENT 16
#define portPOINTER_SIZE_TYPE uint64_t
/*-----------------------------------------------------------*/
/* Task utilities. */
/* Called at the end of an ISR that can cause a context switch. */
#define portEND_SWITCHING_ISR( xSwitchRequired ) \
{ \
extern uint64_t ullPortYieldRequired; \
\
if( xSwitchRequired != pdFALSE ) \
{ \
ullPortYieldRequired = pdTRUE; \
} \
}
#define portYIELD_FROM_ISR( x ) portEND_SWITCHING_ISR( x )
#if defined( GUEST )
#define portYIELD() __asm volatile ( "SVC 0" ::: "memory" )
#else
#define portYIELD() __asm volatile ( "SMC 0" ::: "memory" )
#endif
/*-----------------------------------------------------------
* Critical section control
*----------------------------------------------------------*/
extern void vPortEnterCritical( void );
extern void vPortExitCritical( void );
extern UBaseType_t uxPortSetInterruptMask( void );
extern void vPortClearInterruptMask( UBaseType_t uxNewMaskValue );
extern void vPortInstallFreeRTOSVectorTable( void );
/* Port variables accessed from portASM.S. */
extern volatile uint64_t ullCriticalNesting;
extern uint64_t ullPortTaskHasFPUContext;
extern uint64_t ullPortYieldRequired;
extern uint64_t ullPortInterruptNesting;
extern const uint64_t ullMaxAPIPriorityMask;
#define portDISABLE_INTERRUPTS() \
__asm volatile ( "MSR DAIFSET, #2" ::: "memory" ); \
__asm volatile ( "DSB SY" ); \
__asm volatile ( "ISB SY" );
#define portENABLE_INTERRUPTS() \
__asm volatile ( "MSR DAIFCLR, #2" ::: "memory" ); \
__asm volatile ( "DSB SY" ); \
__asm volatile ( "ISB SY" );
/* These macros do not globally disable/enable interrupts. They do mask off
* interrupts that have a priority below configMAX_API_CALL_INTERRUPT_PRIORITY. */
#define portENTER_CRITICAL() vPortEnterCritical();
#define portEXIT_CRITICAL() vPortExitCritical();
#define portSET_INTERRUPT_MASK_FROM_ISR() uxPortSetInterruptMask()
#define portCLEAR_INTERRUPT_MASK_FROM_ISR( x ) vPortClearInterruptMask( x )
/*-----------------------------------------------------------*/
/* Task function macros as described on the FreeRTOS.org WEB site. These are
* not required for this port but included in case common demo code that uses these
* macros is used. */
#define portTASK_FUNCTION_PROTO( vFunction, pvParameters ) void vFunction( void * pvParameters )
#define portTASK_FUNCTION( vFunction, pvParameters ) void vFunction( void * pvParameters )
/* Prototype of the FreeRTOS tick handler. This must be installed as the
* handler for whichever peripheral is used to generate the RTOS tick. */
void FreeRTOS_Tick_Handler( void );
/* If configUSE_TASK_FPU_SUPPORT is set to 1 (or left undefined) then tasks are
* created without an FPU context and must call vPortTaskUsesFPU() to give
* themselves an FPU context before using any FPU instructions. If
* configUSE_TASK_FPU_SUPPORT is set to 2 then all tasks will have an FPU context
* by default. */
#ifndef configUSE_TASK_FPU_SUPPORT
#define configUSE_TASK_FPU_SUPPORT 1
#endif
#if ( configUSE_TASK_FPU_SUPPORT != 2 )
void vPortTaskUsesFPU( void );
#else
/* Each task has an FPU context already, so define this function away to
* nothing to prevent it from being called accidentally. */
#define vPortTaskUsesFPU()
#endif
#define portTASK_USES_FLOATING_POINT() vPortTaskUsesFPU()
#define portLOWEST_INTERRUPT_PRIORITY ( ( ( uint32_t ) configUNIQUE_INTERRUPT_PRIORITIES ) - 1UL )
#define portLOWEST_USABLE_INTERRUPT_PRIORITY ( portLOWEST_INTERRUPT_PRIORITY - 1UL )
/* Architecture specific optimisations. */
#ifndef configUSE_PORT_OPTIMISED_TASK_SELECTION
#define configUSE_PORT_OPTIMISED_TASK_SELECTION 1
#endif
#if configUSE_PORT_OPTIMISED_TASK_SELECTION == 1
/* Store/clear the ready priorities in a bit map. */
#define portRECORD_READY_PRIORITY( uxPriority, uxReadyPriorities ) ( uxReadyPriorities ) |= ( 1UL << ( uxPriority ) )
#define portRESET_READY_PRIORITY( uxPriority, uxReadyPriorities ) ( uxReadyPriorities ) &= ~( 1UL << ( uxPriority ) )
/*-----------------------------------------------------------*/
#define portGET_HIGHEST_PRIORITY( uxTopPriority, uxReadyPriorities ) uxTopPriority = ( 31 - __builtin_clz( uxReadyPriorities ) )
#endif /* configUSE_PORT_OPTIMISED_TASK_SELECTION */
#if ( configASSERT_DEFINED == 1 )
void vPortValidateInterruptPriority( void );
#define portASSERT_IF_INTERRUPT_PRIORITY_INVALID() vPortValidateInterruptPriority()
#endif /* configASSERT */
#define portNOP() __asm volatile ( "NOP" )
#define portINLINE __inline
/* The number of bits to shift for an interrupt priority is dependent on the
* number of bits implemented by the interrupt controller. */
#if configUNIQUE_INTERRUPT_PRIORITIES == 16
#define portPRIORITY_SHIFT 4
#define portMAX_BINARY_POINT_VALUE 3
#elif configUNIQUE_INTERRUPT_PRIORITIES == 32
#define portPRIORITY_SHIFT 3
#define portMAX_BINARY_POINT_VALUE 2
#elif configUNIQUE_INTERRUPT_PRIORITIES == 64
#define portPRIORITY_SHIFT 2
#define portMAX_BINARY_POINT_VALUE 1
#elif configUNIQUE_INTERRUPT_PRIORITIES == 128
#define portPRIORITY_SHIFT 1
#define portMAX_BINARY_POINT_VALUE 0
#elif configUNIQUE_INTERRUPT_PRIORITIES == 256
#define portPRIORITY_SHIFT 0
#define portMAX_BINARY_POINT_VALUE 0
#else /* if configUNIQUE_INTERRUPT_PRIORITIES == 16 */
#error Invalid configUNIQUE_INTERRUPT_PRIORITIES setting. configUNIQUE_INTERRUPT_PRIORITIES must be set to the number of unique priorities implemented by the target hardware
#endif /* if configUNIQUE_INTERRUPT_PRIORITIES == 16 */
#define portMEMORY_BARRIER() __asm volatile ( "" ::: "memory" )
/* ---- Armv9 MTE heap/stack tagging ---- */
#if ( configARMV9_MTE_STACK == 1 ) || ( configARMV9_MTE_HEAP == 1 )
#define portSTRIP_ADDRESS_TAG( pxPointer ) \
( ( void * )( ( uint64_t )( pxPointer ) & 0x00FFFFFFFFFFFFFFULL ) )
#endif
#if ( configARMV9_MTE_STACK == 1 )
void vPortMteTagStack( StackType_t *pxStack, uint32_t ulStackDepth );
#endif
#if ( configARMV9_MTE_HEAP == 1 )
void *pvPortMallocTagged( size_t xSize );
void vPortFreeTagged( void *pv );
/* Transparent redirect: all pvPortMalloc/vPortFree calls get MTE tagging.
* Excluded from allocator implementation files via PORTMEMORY_IMPLEMENTATION.
*
* MISRA Ref 5.8.1 [Identifier with external linkage reused as macro]
* Rationale: The macro redirect keeps all MTE logic isolated in the port
* directory without modifying shared upstream kernel files (heap_4.c).
* An alternative that would eliminate this deviation is to integrate the
* MTE tagging directly into heap_4.c, guarded by configARMV9_MTE_HEAP.
* That approach trades this violation for a modification to a shared file
* with associated rebase/merge risk and cross-platform #include concerns. */
#if !defined( PORTMEMORY_IMPLEMENTATION )
#define pvPortMalloc pvPortMallocTagged
#define vPortFree vPortFreeTagged
#endif
#endif
/* ---- Armv9 PAC per-task key management ---- */
#if ( configARMV9_PAC == 1 )
void vPortTaskRegeneratePACKeys( void );
#endif
/* ---- Armv9 per-task SVE vector length ---- */
#if ( configARMV9_TASK_VL == 1 )
void vPortTaskSetVL( uint32_t ulVL );
#endif
/* *INDENT-OFF* */
#ifdef __cplusplus
}
#endif
/* *INDENT-ON* */
#endif /* PORTMACRO_H */

14
tasks.c
View file

@ -2015,11 +2015,21 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode,
#if ( portSTACK_GROWTH < 0 )
{
configASSERT( ( ( portPOINTER_SIZE_TYPE ) ( pxTopOfStack - pxNewTCB->pxTopOfStack ) ) < ( ( portPOINTER_SIZE_TYPE ) uxStackDepth ) );
/* portSTRIP_ADDRESS_TAG handles MTE-tagged pointers returned by
* pxPortInitialiseStack when configARMV9_MTE_STACK is enabled. */
#ifdef portSTRIP_ADDRESS_TAG
configASSERT( ( ( portPOINTER_SIZE_TYPE ) ( portSTRIP_ADDRESS_TAG( pxTopOfStack ) - portSTRIP_ADDRESS_TAG( pxNewTCB->pxTopOfStack ) ) ) < ( ( portPOINTER_SIZE_TYPE ) uxStackDepth ) );
#else
configASSERT( ( ( portPOINTER_SIZE_TYPE ) ( pxTopOfStack - pxNewTCB->pxTopOfStack ) ) < ( ( portPOINTER_SIZE_TYPE ) uxStackDepth ) );
#endif
}
#else /* portSTACK_GROWTH */
{
configASSERT( ( ( portPOINTER_SIZE_TYPE ) ( pxNewTCB->pxTopOfStack - pxTopOfStack ) ) < ( ( portPOINTER_SIZE_TYPE ) uxStackDepth ) );
#ifdef portSTRIP_ADDRESS_TAG
configASSERT( ( ( portPOINTER_SIZE_TYPE ) ( portSTRIP_ADDRESS_TAG( pxNewTCB->pxTopOfStack ) - portSTRIP_ADDRESS_TAG( pxTopOfStack ) ) ) < ( ( portPOINTER_SIZE_TYPE ) uxStackDepth ) );
#else
configASSERT( ( ( portPOINTER_SIZE_TYPE ) ( pxNewTCB->pxTopOfStack - pxTopOfStack ) ) < ( ( portPOINTER_SIZE_TYPE ) uxStackDepth ) );
#endif
}
#endif /* portSTACK_GROWTH */
}