From 7fa93024a1e42be102309af2bf89580628eacf0e Mon Sep 17 00:00:00 2001 From: Darian Leung Date: Sun, 16 Jun 2024 00:48:25 +0800 Subject: [PATCH] feat(granular_locks): Add granular locking functions - Updated prvCheckForRunStateChange() for granular locks - Updated vTaskSuspendAll() and xTaskResumeAll() - Now holds the xTaskSpinlock during kernel suspension - Increments/decrements xPreemptionDisable. Only yields when 0, thus allowing for nested suspensions across different data groups Co-authored-by: Sudeep Mohanty --- include/FreeRTOS.h | 118 +++----------------------- include/task.h | 201 ++++++++++++++++++++++++++++++++++++++++++--- tasks.c | 127 ++++++++++++++++++++-------- 3 files changed, 293 insertions(+), 153 deletions(-) diff --git a/include/FreeRTOS.h b/include/FreeRTOS.h index d3a76c79a..efdcf9e0a 100644 --- a/include/FreeRTOS.h +++ b/include/FreeRTOS.h @@ -465,7 +465,7 @@ #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) ) #define portRELEASE_TASK_LOCK( xCoreID ) #else - #error portRELEASE_TASK_LOCK is required in SMP + #error portRELEASE_TASK_LOCK is required in SMP without granular locking feature enabled #endif #endif /* portRELEASE_TASK_LOCK */ @@ -475,7 +475,7 @@ #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) ) #define portGET_TASK_LOCK( xCoreID ) #else - #error portGET_TASK_LOCK is required in SMP + #error portGET_TASK_LOCK is required in SMP without granular locking feature enabled #endif #endif /* portGET_TASK_LOCK */ @@ -485,7 +485,7 @@ #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) ) #define portRELEASE_ISR_LOCK( xCoreID ) #else - #error portRELEASE_ISR_LOCK is required in SMP + #error portRELEASE_ISR_LOCK is required in SMP without granular locking feature enabled #endif #endif /* portRELEASE_ISR_LOCK */ @@ -495,7 +495,7 @@ #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) ) #define portGET_ISR_LOCK( xCoreID ) #else - #error portGET_ISR_LOCK is required in SMP + #error portGET_ISR_LOCK is required in SMP without granular locking feature enabled #endif #endif /* portGET_ISR_LOCK */ @@ -503,7 +503,7 @@ #ifndef portRELEASE_SPINLOCK #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portRELEASE_SPINLOCK is required for granular locking + #error portRELEASE_SPINLOCK is required for SMP with granular locking feature enabled #endif #endif @@ -511,7 +511,7 @@ #ifndef portGET_SPINLOCK #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portGET_SPINLOCK is required for granular locking + #error portGET_SPINLOCK is required for SMP with granular locking feature enabled #endif #endif @@ -540,38 +540,6 @@ #endif -#ifndef portENTER_CRITICAL_DATA_GROUP - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portENTER_CRITICAL_DATA_GROUP is required for SMP with granular locking feature enabled - #endif - -#endif - -#ifndef portEXIT_CRITICAL_DATA_GROUP - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portEXIT_CRITICAL_DATA_GROUP is required for SMP with granular locking feature enabled - #endif - -#endif - -#ifndef portENTER_CRITICAL_DATA_GROUP_FROM_ISR - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portENTER_CRITICAL_DATA_GROUP_FROM_ISR is required for SMP with granular locking feature enabled - #endif - -#endif - -#ifndef portEXIT_CRITICAL_DATA_GROUP_FROM_ISR - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portEXIT_CRITICAL_DATA_GROUP_FROM_ISR is required for SMP with granular locking feature enabled - #endif - -#endif - #ifndef portSPINLOCK_TYPE #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) @@ -580,82 +548,18 @@ #endif -#ifndef portINIT_EVENT_GROUP_TASK_SPINLOCK +#ifndef portINIT_SPINLOCK #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_EVENT_GROUP_TASK_SPINLOCK is required for granular locking + #error portINIT_SPINLOCK is required for SMP with granular locking feature enabled #endif #endif -#ifndef portINIT_EVENT_GROUP_ISR_SPINLOCK +#ifndef portINIT_SPINLOCK_STATIC #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_EVENT_GROUP_ISR_SPINLOCK is required for granular locking - #endif - -#endif - -#ifndef portINIT_QUEUE_TASK_SPINLOCK - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_QUEUE_TASK_SPINLOCK is required for granular locking - #endif - -#endif - -#ifndef portINIT_QUEUE_ISR_SPINLOCK - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_QUEUE_ISR_SPINLOCK is required for granular locking - #endif - -#endif - -#ifndef portINIT_STREAM_BUFFER_TASK_SPINLOCK - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_STREAM_BUFFER_TASK_SPINLOCK is required for granular locking - #endif - -#endif - -#ifndef portINIT_STREAM_BUFFER_ISR_SPINLOCK - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_STREAM_BUFFER_ISR_SPINLOCK is required for granular locking - #endif - -#endif - -#ifndef portINIT_KERNEL_TASK_SPINLOCK_STATIC - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_KERNEL_TASK_SPINLOCK_STATIC is required for granular locking - #endif - -#endif - -#ifndef portINIT_KERNEL_ISR_SPINLOCK_STATIC - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_KERNEL_ISR_SPINLOCK_STATIC is required for granular locking - #endif - -#endif - -#ifndef portINIT_TIMERS_TASK_SPINLOCK_STATIC - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_TIMERS_TASK_SPINLOCK_STATIC is required for granular locking - #endif - -#endif - -#ifndef portINIT_TIMERS_ISR_SPINLOCK_STATIC - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_TIMERS_ISR_SPINLOCK_STATIC is required for granular locking + #error portINIT_SPINLOCK_STATIC is required for SMP with granular locking feature enabled #endif #endif @@ -3090,7 +2994,7 @@ #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ #define portTICK_TYPE_SET_INTERRUPT_MASK_FROM_ISR() portSET_INTERRUPT_MASK_FROM_ISR() #define portTICK_TYPE_CLEAR_INTERRUPT_MASK_FROM_ISR( x ) portCLEAR_INTERRUPT_MASK_FROM_ISR( ( x ) ) -#else +#else /* if ( portTICK_TYPE_IS_ATOMIC == 0 ) */ /* The tick type can be read atomically, so critical sections used when the * tick count is returned can be defined away. */ diff --git a/include/task.h b/include/task.h index 193d242f5..3fa505b05 100644 --- a/include/task.h +++ b/include/task.h @@ -224,7 +224,7 @@ typedef enum * \defgroup taskYIELD taskYIELD * \ingroup SchedulerControl */ -#define taskYIELD() portYIELD() +#define taskYIELD() portYIELD() /** * task. h @@ -238,12 +238,19 @@ typedef enum * \defgroup taskENTER_CRITICAL taskENTER_CRITICAL * \ingroup SchedulerControl */ -#define taskENTER_CRITICAL() portENTER_CRITICAL() +#define taskENTER_CRITICAL() portENTER_CRITICAL() #if ( configNUMBER_OF_CORES == 1 ) - #define taskENTER_CRITICAL_FROM_ISR() portSET_INTERRUPT_MASK_FROM_ISR() + #define taskENTER_CRITICAL_FROM_ISR() portSET_INTERRUPT_MASK_FROM_ISR() #else - #define taskENTER_CRITICAL_FROM_ISR() portENTER_CRITICAL_FROM_ISR() + #define taskENTER_CRITICAL_FROM_ISR() portENTER_CRITICAL_FROM_ISR() #endif +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #define taskLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) portLOCK_DATA_GROUP( ( portSPINLOCK_TYPE * ) pxTaskSpinlock, ( portSPINLOCK_TYPE * ) pxISRSpinlock ) + #define taskLOCK_DATA_GROUP_FROM_ISR( pxISRSpinlock ) portLOCK_DATA_GROUP_FROM_ISR( pxISRSpinlock ) +#else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #define taskLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) taskENTER_CRITICAL() + #define taskLOCK_DATA_GROUP_FROM_ISR( pxISRSpinlock ) taskENTER_CRITICAL_FROM_ISR() +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /** * task. h @@ -257,12 +264,19 @@ typedef enum * \defgroup taskEXIT_CRITICAL taskEXIT_CRITICAL * \ingroup SchedulerControl */ -#define taskEXIT_CRITICAL() portEXIT_CRITICAL() +#define taskEXIT_CRITICAL() portEXIT_CRITICAL() #if ( configNUMBER_OF_CORES == 1 ) - #define taskEXIT_CRITICAL_FROM_ISR( x ) portCLEAR_INTERRUPT_MASK_FROM_ISR( x ) + #define taskEXIT_CRITICAL_FROM_ISR( x ) portCLEAR_INTERRUPT_MASK_FROM_ISR( x ) #else - #define taskEXIT_CRITICAL_FROM_ISR( x ) portEXIT_CRITICAL_FROM_ISR( x ) + #define taskEXIT_CRITICAL_FROM_ISR( x ) portEXIT_CRITICAL_FROM_ISR( x ) #endif +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #define taskUNLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) portUNLOCK_DATA_GROUP( ( portSPINLOCK_TYPE * ) pxTaskSpinlock, ( portSPINLOCK_TYPE * ) pxISRSpinlock ) + #define taskUNLOCK_DATA_GROUP_FROM_ISR( x, pxISRSpinlock ) portUNLOCK_DATA_GROUP_FROM_ISR( x, pxISRSpinlock ) +#else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #define taskUNLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) taskEXIT_CRITICAL() + #define taskUNLOCK_DATA_GROUP_FROM_ISR( x, pxISRSpinlock ) taskEXIT_CRITICAL_FROM_ISR( x ) +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /** * task. h @@ -294,6 +308,146 @@ typedef enum /* Checks if core ID is valid. */ #define taskVALID_CORE_ID( xCoreID ) ( ( ( ( ( BaseType_t ) 0 <= ( xCoreID ) ) && ( ( xCoreID ) < ( BaseType_t ) configNUMBER_OF_CORES ) ) ) ? ( pdTRUE ) : ( pdFALSE ) ) +/** + * task. h + * + * Macro to enter a data group critical section. + * + * \defgroup taskDATA_GROUP_ENTER_CRITICAL taskDATA_GROUP_ENTER_CRITICAL + * \ingroup GranularLocks + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define taskDATA_GROUP_ENTER_CRITICAL( pxTaskSpinlock, pxISRSpinlock ) \ + do { \ + /* Disable preemption to avoid task state changes during the critical section. */ \ + vTaskPreemptionDisable( NULL ); \ + { \ + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ + /* Task spinlock is always taken first */ \ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxTaskSpinlock ); \ + /* Disable interrupts */ \ + portDISABLE_INTERRUPTS(); \ + /* Take the ISR spinlock next */ \ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock ); \ + /* Increment the critical nesting count */ \ + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ + } \ + } while( 0 ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + +/** + * task. h + * + * Macro to enter a data group critical section from an interrupt. + * + * \defgroup taskDATA_GROUP_ENTER_CRITICAL_FROM_ISR taskDATA_GROUP_ENTER_CRITICAL_FROM_ISR + * \ingroup GranularLocks + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define taskDATA_GROUP_ENTER_CRITICAL_FROM_ISR( pxISRSpinlock, puxSavedInterruptStatus ) \ + do { \ + *( puxSavedInterruptStatus ) = portSET_INTERRUPT_MASK_FROM_ISR(); \ + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ + /* Take the ISR spinlock */ \ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock ); \ + /* Increment the critical nesting count */ \ + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ + } while( 0 ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + +/** + * task. h + * + * Macro to exit a data group critical section. + * + * \defgroup taskDATA_GROUP_EXIT_CRITICAL taskDATA_GROUP_EXIT_CRITICAL + * \ingroup GranularLocks + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define taskDATA_GROUP_EXIT_CRITICAL( pxTaskSpinlock, pxISRSpinlock ) \ + do { \ + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ + configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ); \ + /* Release the ISR spinlock */ \ + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock ); \ + /* Release the task spinlock */ \ + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxTaskSpinlock ); \ + /* Decrement the critical nesting count */ \ + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ + /* Enable interrupts only if the critical nesting count is 0 */ \ + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 ) \ + { \ + portENABLE_INTERRUPTS(); \ + } \ + else \ + { \ + mtCOVERAGE_TEST_MARKER(); \ + } \ + /* Re-enable preemption */ \ + vTaskPreemptionEnable( NULL ); \ + } while( 0 ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + +/** + * task. h + * + * Macro to exit a data group critical section from an interrupt. + * + * \defgroup taskDATA_GROUP_EXIT_CRITICAL_FROM_ISR taskDATA_GROUP_EXIT_CRITICAL_FROM_ISR + * \ingroup GranularLocks + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define taskDATA_GROUP_EXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxISRSpinlock ) \ + do { \ + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ + configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ); \ + /* Decrement the critical nesting count */ \ + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ + /* Release the ISR spinlock */ \ + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock ); \ + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 ) \ + { \ + portCLEAR_INTERRUPT_MASK_FROM_ISR( uxSavedInterruptStatus ); \ + } \ + } while( 0 ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + +/** + * task. h + * + * Macros to lock a data group (task-level lock only). + * + * \defgroup taskDATA_GROUP_LOCK taskDATA_GROUP_LOCK + * \ingroup GranularLocks + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define taskDATA_GROUP_LOCK( pxTaskSpinlock ) \ + do { \ + /* Disable preemption while holding the task spinlock. */ \ + vTaskPreemptionDisable( NULL ); \ + { \ + portGET_SPINLOCK( portGET_CORE_ID(), ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ); \ + } \ + } while( 0 ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + +/** + * task. h + * + * Macros to unlock a data group (task-level lock only). + * + * \defgroup taskDATA_GROUP_UNLOCK taskDATA_GROUP_UNLOCK + * \ingroup GranularLocks + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define taskDATA_GROUP_UNLOCK( pxTaskSpinlock ) \ + ( { \ + portRELEASE_SPINLOCK( portGET_CORE_ID(), ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ); \ + /* Re-enable preemption after releasing the task spinlock. */ \ + prvTaskPreemptionEnable( NULL ); \ + } ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + /*----------------------------------------------------------- * TASK CREATION API *----------------------------------------------------------*/ @@ -1556,6 +1710,23 @@ BaseType_t xTaskResumeFromISR( TaskHandle_t xTaskToResume ) PRIVILEGED_FUNCTION; void vTaskPreemptionEnable( const TaskHandle_t xTask ); #endif +#if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + +/* + * THIS FUNCTION MUST NOT BE USED FROM APPLICATION CODE. IT IS ONLY + * INTENDED FOR USE WHEN IMPLEMENTING A PORT OF THE SCHEDULER AND IS + * AN INTERFACE WHICH IS FOR THE EXCLUSIVE USE OF THE SCHEDULER. + * + * @param xTask The handle of the task to enable preemption. Passing NULL + * enables preemption for the calling task. + * + * @return pdTRUE if enabling preemption for the task resulted in a context + * switch, otherwise pdFALSE. This is used by the scheduler to determine if a + * context switch may be required following the enable. + */ + BaseType_t prvTaskPreemptionEnable( const TaskHandle_t xTask ); +#endif + /*----------------------------------------------------------- * SCHEDULER CONTROL *----------------------------------------------------------*/ @@ -3825,7 +3996,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC * It should be used in the implementation of portENTER_CRITICAL if port is running a * multiple core FreeRTOS. */ -#if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) || ( configNUMBER_OF_CORES > 1 ) ) +#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) void vTaskEnterCritical( void ); #endif @@ -3837,7 +4008,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC * It should be used in the implementation of portEXIT_CRITICAL if port is running a * multiple core FreeRTOS. */ -#if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) || ( configNUMBER_OF_CORES > 1 ) ) +#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) void vTaskExitCritical( void ); #endif @@ -3847,7 +4018,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC * should be used in the implementation of portENTER_CRITICAL_FROM_ISR if port is * running a multiple core FreeRTOS. */ -#if ( configNUMBER_OF_CORES > 1 ) +#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) UBaseType_t vTaskEnterCriticalFromISR( void ); #endif @@ -3857,10 +4028,18 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC * should be used in the implementation of portEXIT_CRITICAL_FROM_ISR if port is * running a multiple core FreeRTOS. */ -#if ( configNUMBER_OF_CORES > 1 ) +#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) void vTaskExitCriticalFromISR( UBaseType_t uxSavedInterruptStatus ); #endif +/* + * Checks whether a yield is required after taskUNLOCK_DATA_GROUP() returns. + * To be called while data group is locked. + */ +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + BaseType_t xTaskUnlockCanYield( void ); +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #if ( portUSING_MPU_WRAPPERS == 1 ) /* diff --git a/tasks.c b/tasks.c index c0ce16228..97b18b041 100644 --- a/tasks.c +++ b/tasks.c @@ -541,6 +541,11 @@ PRIVILEGED_DATA STATIC volatile configRUN_TIME_COUNTER_TYPE ulTotalRunTime[ conf #endif +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + PRIVILEGED_DATA static portSPINLOCK_TYPE xTaskSpinlock = portINIT_KERNEL_TASK_SPINLOCK_STATIC; + PRIVILEGED_DATA static portSPINLOCK_TYPE xISRSpinlock = portINIT_KERNEL_ISR_SPINLOCK_STATIC; +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + /*-----------------------------------------------------------*/ /* File private functions. --------------------------------*/ @@ -550,14 +555,14 @@ PRIVILEGED_DATA STATIC volatile configRUN_TIME_COUNTER_TYPE ulTotalRunTime[ conf */ STATIC BaseType_t prvCreateIdleTasks( void ); -#if ( configNUMBER_OF_CORES > 1 ) +#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) /* * Checks to see if another task moved the current task out of the ready * list while it was waiting to enter a critical section and yields, if so. */ STATIC void prvCheckForRunStateChange( void ); -#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ #if ( configNUMBER_OF_CORES > 1 ) @@ -807,7 +812,7 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; #endif /* #if ( ( configUSE_TRACE_FACILITY == 1 ) && ( configUSE_STATS_FORMATTING_FUNCTIONS > 0 ) ) */ /*-----------------------------------------------------------*/ -#if ( configNUMBER_OF_CORES > 1 ) +#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) STATIC void prvCheckForRunStateChange( void ) { UBaseType_t uxPrevCriticalNesting; @@ -871,7 +876,7 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; } } } -#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /*-----------------------------------------------------------*/ @@ -3958,31 +3963,45 @@ void vTaskSuspendAll( void ) * do not otherwise exhibit real time behaviour. */ portSOFTWARE_BARRIER(); - portGET_TASK_LOCK( xCoreID ); - - /* uxSchedulerSuspended is increased after prvCheckForRunStateChange. The - * purpose is to prevent altering the variable when fromISR APIs are readying - * it. */ - if( uxSchedulerSuspended == 0U ) + #if ( portUSING_GRANULAR_LOCKS == 1 ) { - prvCheckForRunStateChange(); + portGET_SPINLOCK( &xTaskSpinlock ); + portGET_SPINLOCK( &xISRSpinlock ); + + /* Increment xPreemptionDisable to prevent preemption and also + * track whether to yield in xTaskResumeAll(). */ + pxCurrentTCBs[ portGET_CORE_ID() ]->xPreemptionDisable++; + + /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment + * is used to allow calls to vTaskSuspendAll() to nest. */ + ++uxSchedulerSuspended; + + portRELEASE_SPINLOCK( &xISRSpinlock ); } - else + #else /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ { - mtCOVERAGE_TEST_MARKER(); + portGET_TASK_LOCK(); + + /* uxSchedulerSuspended is increased after prvCheckForRunStateChange. The + * purpose is to prevent altering the variable when fromISR APIs are readying + * it. */ + if( uxSchedulerSuspended == 0U ) + { + prvCheckForRunStateChange(); + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + + portGET_ISR_LOCK(); + + /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment + * is used to allow calls to vTaskSuspendAll() to nest. */ + ++uxSchedulerSuspended; + portRELEASE_ISR_LOCK(); } - - /* Query the coreID again as prvCheckForRunStateChange may have - * caused the task to get scheduled on a different core. The correct - * task lock for the core is acquired in prvCheckForRunStateChange. */ - xCoreID = ( BaseType_t ) portGET_CORE_ID(); - - portGET_ISR_LOCK( xCoreID ); - - /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment - * is used to allow calls to vTaskSuspendAll() to nest. */ - ++uxSchedulerSuspended; - portRELEASE_ISR_LOCK( xCoreID ); + #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ portCLEAR_INTERRUPT_MASK( ulState ); } @@ -4087,7 +4106,24 @@ BaseType_t xTaskResumeAll( void ) configASSERT( uxSchedulerSuspended != 0U ); uxSchedulerSuspended = ( UBaseType_t ) ( uxSchedulerSuspended - 1U ); - portRELEASE_TASK_LOCK( xCoreID ); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + { + configASSERT( pxCurrentTCBs[ portGET_CORE_ID() ]->xPreemptionDisable > 0 ); + + /* Decrement xPreemptionDisable. If 0, it means this we are not + * in a nested suspension scenario, thus this function and yield + * if necessary. */ + pxCurrentTCBs[ portGET_CORE_ID() ]->xPreemptionDisable--; + + /* Release the kernel's task spinlock that was held throughout + * the kernel suspension. */ + portRELEASE_SPINLOCK( &xTaskSpinlock ); + } + #else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + { + portRELEASE_TASK_LOCK(); + } + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ if( uxSchedulerSuspended == ( UBaseType_t ) 0U ) { @@ -7129,7 +7165,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) #endif /* #if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) && ( configNUMBER_OF_CORES == 1 ) ) */ /*-----------------------------------------------------------*/ -#if ( configNUMBER_OF_CORES > 1 ) +#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) void vTaskEnterCritical( void ) { @@ -7178,11 +7214,10 @@ STATIC void prvResetNextTaskUnblockTime( void ) traceRETURN_vTaskEnterCritical(); } -#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ - +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /*-----------------------------------------------------------*/ -#if ( configNUMBER_OF_CORES > 1 ) +#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) UBaseType_t vTaskEnterCriticalFromISR( void ) { @@ -7212,7 +7247,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) return uxSavedInterruptStatus; } -#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /*-----------------------------------------------------------*/ #if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) && ( configNUMBER_OF_CORES == 1 ) ) @@ -7260,7 +7295,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) #endif /* #if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) && ( configNUMBER_OF_CORES == 1 ) ) */ /*-----------------------------------------------------------*/ -#if ( configNUMBER_OF_CORES > 1 ) +#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) void vTaskExitCritical( void ) { @@ -7320,10 +7355,10 @@ STATIC void prvResetNextTaskUnblockTime( void ) traceRETURN_vTaskExitCritical(); } -#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /*-----------------------------------------------------------*/ -#if ( configNUMBER_OF_CORES > 1 ) +#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) void vTaskExitCriticalFromISR( UBaseType_t uxSavedInterruptStatus ) { @@ -7366,7 +7401,29 @@ STATIC void prvResetNextTaskUnblockTime( void ) traceRETURN_vTaskExitCriticalFromISR(); } -#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ +/*-----------------------------------------------------------*/ + +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + + BaseType_t xTaskUnlockCanYield( void ) + { + BaseType_t xReturn; + BaseType_t xCoreID = portGET_CORE_ID(); + + if( ( xYieldPendings[ xCoreID ] == pdTRUE ) && ( uxSchedulerSuspended == pdFALSE ) && ( pxCurrentTCBs[ xCoreID ]->xPreemptionDisable == pdFALSE ) ) + { + xReturn = pdTRUE; + } + else + { + xReturn = pdFALSE; + } + + return xReturn; + } + +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /*-----------------------------------------------------------*/ #if ( configUSE_STATS_FORMATTING_FUNCTIONS > 0 )