From 4caec37453aa316d78c01c80c26de60af9302b58 Mon Sep 17 00:00:00 2001 From: Darian Leung Date: Sun, 16 Jun 2024 00:31:09 +0800 Subject: [PATCH 01/11] refactor(freertos/smp): Move critical sections inside xTaskPriorityInherit() xTaskPriorityInherit() is called inside a critical section from queue.c. This commit moves the critical section into xTaskPriorityInherit(). Co-authored-by: Sudeep Mohanty --- queue.c | 6 +-- tasks.c | 128 +++++++++++++++++++++++++++++--------------------------- 2 files changed, 67 insertions(+), 67 deletions(-) diff --git a/queue.c b/queue.c index 37d953ac9..8df1481e8 100644 --- a/queue.c +++ b/queue.c @@ -1790,11 +1790,7 @@ BaseType_t xQueueSemaphoreTake( QueueHandle_t xQueue, { if( pxQueue->uxQueueType == queueQUEUE_IS_MUTEX ) { - taskENTER_CRITICAL(); - { - xInheritanceOccurred = xTaskPriorityInherit( pxQueue->u.xSemaphore.xMutexHolder ); - } - taskEXIT_CRITICAL(); + xInheritanceOccurred = xTaskPriorityInherit( pxQueue->u.xSemaphore.xMutexHolder ); } else { diff --git a/tasks.c b/tasks.c index 461271fcf..c16f13f1d 100644 --- a/tasks.c +++ b/tasks.c @@ -6665,91 +6665,95 @@ STATIC void prvResetNextTaskUnblockTime( void ) traceENTER_xTaskPriorityInherit( pxMutexHolder ); - /* If the mutex is taken by an interrupt, the mutex holder is NULL. Priority - * inheritance is not applied in this scenario. */ - if( pxMutexHolder != NULL ) + taskENTER_CRITICAL(); { - /* If the holder of the mutex has a priority below the priority of - * the task attempting to obtain the mutex then it will temporarily - * inherit the priority of the task attempting to obtain the mutex. */ - if( pxMutexHolderTCB->uxPriority < pxCurrentTCB->uxPriority ) + /* If the mutex is taken by an interrupt, the mutex holder is NULL. Priority + * inheritance is not applied in this scenario. */ + if( pxMutexHolder != NULL ) { - /* Adjust the mutex holder state to account for its new - * priority. Only reset the event list item value if the value is - * not being used for anything else. */ - if( ( listGET_LIST_ITEM_VALUE( &( pxMutexHolderTCB->xEventListItem ) ) & taskEVENT_LIST_ITEM_VALUE_IN_USE ) == ( ( TickType_t ) 0U ) ) + /* If the holder of the mutex has a priority below the priority of + * the task attempting to obtain the mutex then it will temporarily + * inherit the priority of the task attempting to obtain the mutex. */ + if( pxMutexHolderTCB->uxPriority < pxCurrentTCB->uxPriority ) { - listSET_LIST_ITEM_VALUE( &( pxMutexHolderTCB->xEventListItem ), ( TickType_t ) configMAX_PRIORITIES - ( TickType_t ) pxCurrentTCB->uxPriority ); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } - - /* If the task being modified is in the ready state it will need - * to be moved into a new list. */ - if( listIS_CONTAINED_WITHIN( &( pxReadyTasksLists[ pxMutexHolderTCB->uxPriority ] ), &( pxMutexHolderTCB->xStateListItem ) ) != pdFALSE ) - { - if( uxListRemove( &( pxMutexHolderTCB->xStateListItem ) ) == ( UBaseType_t ) 0 ) + /* Adjust the mutex holder state to account for its new + * priority. Only reset the event list item value if the value is + * not being used for anything else. */ + if( ( listGET_LIST_ITEM_VALUE( &( pxMutexHolderTCB->xEventListItem ) ) & taskEVENT_LIST_ITEM_VALUE_IN_USE ) == ( ( TickType_t ) 0U ) ) { - /* It is known that the task is in its ready list so - * there is no need to check again and the port level - * reset macro can be called directly. */ - portRESET_READY_PRIORITY( pxMutexHolderTCB->uxPriority, uxTopReadyPriority ); + listSET_LIST_ITEM_VALUE( &( pxMutexHolderTCB->xEventListItem ), ( TickType_t ) configMAX_PRIORITIES - ( TickType_t ) pxCurrentTCB->uxPriority ); } else { mtCOVERAGE_TEST_MARKER(); } - /* Inherit the priority before being moved into the new list. */ - pxMutexHolderTCB->uxPriority = pxCurrentTCB->uxPriority; - prvAddTaskToReadyList( pxMutexHolderTCB ); - #if ( configNUMBER_OF_CORES > 1 ) + /* If the task being modified is in the ready state it will need + * to be moved into a new list. */ + if( listIS_CONTAINED_WITHIN( &( pxReadyTasksLists[ pxMutexHolderTCB->uxPriority ] ), &( pxMutexHolderTCB->xStateListItem ) ) != pdFALSE ) { - /* The priority of the task is raised. Yield for this task - * if it is not running. */ - if( taskTASK_IS_RUNNING( pxMutexHolderTCB ) != pdTRUE ) + if( uxListRemove( &( pxMutexHolderTCB->xStateListItem ) ) == ( UBaseType_t ) 0 ) { - prvYieldForTask( pxMutexHolderTCB ); + /* It is known that the task is in its ready list so + * there is no need to check again and the port level + * reset macro can be called directly. */ + portRESET_READY_PRIORITY( pxMutexHolderTCB->uxPriority, uxTopReadyPriority ); } + else + { + mtCOVERAGE_TEST_MARKER(); + } + + /* Inherit the priority before being moved into the new list. */ + pxMutexHolderTCB->uxPriority = pxCurrentTCB->uxPriority; + prvAddTaskToReadyList( pxMutexHolderTCB ); + #if ( configNUMBER_OF_CORES > 1 ) + { + /* The priority of the task is raised. Yield for this task + * if it is not running. */ + if( taskTASK_IS_RUNNING( pxMutexHolderTCB ) != pdTRUE ) + { + prvYieldForTask( pxMutexHolderTCB ); + } + } + #endif /* if ( configNUMBER_OF_CORES > 1 ) */ + } + else + { + /* Just inherit the priority. */ + pxMutexHolderTCB->uxPriority = pxCurrentTCB->uxPriority; } - #endif /* if ( configNUMBER_OF_CORES > 1 ) */ - } - else - { - /* Just inherit the priority. */ - pxMutexHolderTCB->uxPriority = pxCurrentTCB->uxPriority; - } - traceTASK_PRIORITY_INHERIT( pxMutexHolderTCB, pxCurrentTCB->uxPriority ); + traceTASK_PRIORITY_INHERIT( pxMutexHolderTCB, pxCurrentTCB->uxPriority ); - /* Inheritance occurred. */ - xReturn = pdTRUE; - } - else - { - if( pxMutexHolderTCB->uxBasePriority < pxCurrentTCB->uxPriority ) - { - /* The base priority of the mutex holder is lower than the - * priority of the task attempting to take the mutex, but the - * current priority of the mutex holder is not lower than the - * priority of the task attempting to take the mutex. - * Therefore the mutex holder must have already inherited a - * priority, but inheritance would have occurred if that had - * not been the case. */ + /* Inheritance occurred. */ xReturn = pdTRUE; } else { - mtCOVERAGE_TEST_MARKER(); + if( pxMutexHolderTCB->uxBasePriority < pxCurrentTCB->uxPriority ) + { + /* The base priority of the mutex holder is lower than the + * priority of the task attempting to take the mutex, but the + * current priority of the mutex holder is not lower than the + * priority of the task attempting to take the mutex. + * Therefore the mutex holder must have already inherited a + * priority, but inheritance would have occurred if that had + * not been the case. */ + xReturn = pdTRUE; + } + else + { + mtCOVERAGE_TEST_MARKER(); + } } } + else + { + mtCOVERAGE_TEST_MARKER(); + } } - else - { - mtCOVERAGE_TEST_MARKER(); - } + taskEXIT_CRITICAL(); traceRETURN_xTaskPriorityInherit( xReturn ); From b0f60b2659b69e5bbf414d6c40244fd9c1b6813a Mon Sep 17 00:00:00 2001 From: Darian Leung Date: Sun, 16 Jun 2024 00:37:28 +0800 Subject: [PATCH 02/11] feat(freertos/smp): Allow vTaskPreemptionEnable() to be nested Changed xPreemptionDisable to be a count rather than a pdTRUE/pdFALSE. This allows nested calls to vTaskPreemptionEnable(), where a yield only occurs when xPreemptionDisable is 0. Co-authored-by: Sudeep Mohanty --- include/FreeRTOS.h | 2 +- tasks.c | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/include/FreeRTOS.h b/include/FreeRTOS.h index 83fd430bd..1831c8491 100644 --- a/include/FreeRTOS.h +++ b/include/FreeRTOS.h @@ -3183,7 +3183,7 @@ typedef struct xSTATIC_TCB #endif uint8_t ucDummy7[ configMAX_TASK_NAME_LEN ]; #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - BaseType_t xDummy25; + UBaseType_t xDummy25; #endif #if ( ( portSTACK_GROWTH > 0 ) || ( configRECORD_STACK_HIGH_ADDRESS == 1 ) ) void * pxDummy8; diff --git a/tasks.c b/tasks.c index c16f13f1d..75f03aef7 100644 --- a/tasks.c +++ b/tasks.c @@ -397,7 +397,7 @@ typedef struct tskTaskControlBlock /* The old naming convention is used to char pcTaskName[ configMAX_TASK_NAME_LEN ]; /**< Descriptive name given to the task when created. Facilitates debugging only. */ #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - BaseType_t xPreemptionDisable; /**< Used to prevent the task from being preempted. */ + UBaseType_t xPreemptionDisable; /**< Used to prevent the task from being preempted. */ #endif #if ( ( portSTACK_GROWTH > 0 ) || ( configRECORD_STACK_HIGH_ADDRESS == 1 ) ) @@ -946,7 +946,7 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; #endif { #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - if( pxCurrentTCBs[ xCoreID ]->xPreemptionDisable == pdFALSE ) + if( pxCurrentTCBs[ xCoreID ]->xPreemptionDisable == 0U ) #endif { xLowestPriorityToPreempt = xCurrentCoreTaskPriority; @@ -1252,7 +1252,7 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; ( xYieldPendings[ uxCore ] == pdFALSE ) ) { #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - if( pxCurrentTCBs[ uxCore ]->xPreemptionDisable == pdFALSE ) + if( pxCurrentTCBs[ uxCore ]->xPreemptionDisable == 0U ) #endif { xLowestPriority = xTaskPriority; @@ -2895,7 +2895,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, * there may now be another task of higher priority that * is ready to execute. */ #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - if( pxTCB->xPreemptionDisable == pdFALSE ) + if( pxTCB->xPreemptionDisable == 0U ) #endif { xYieldRequired = pdTRUE; @@ -3116,7 +3116,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, pxTCB = prvGetTCBFromHandle( xTask ); configASSERT( pxTCB != NULL ); - pxTCB->xPreemptionDisable = pdTRUE; + pxTCB->xPreemptionDisable++; } taskEXIT_CRITICAL(); @@ -3139,12 +3139,13 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, { pxTCB = prvGetTCBFromHandle( xTask ); configASSERT( pxTCB != NULL ); + configASSERT( pxTCB->xPreemptionDisable > 0U ); - pxTCB->xPreemptionDisable = pdFALSE; + pxTCB->xPreemptionDisable--; if( xSchedulerRunning != pdFALSE ) { - if( taskTASK_IS_RUNNING( pxTCB ) == pdTRUE ) + if( ( pxTCB->xPreemptionDisable == 0U ) && ( taskTASK_IS_RUNNING( pxTCB ) == pdTRUE ) ) { xCoreID = ( BaseType_t ) pxTCB->xTaskRunState; prvYieldCore( xCoreID ); @@ -4948,7 +4949,7 @@ BaseType_t xTaskIncrementTick( void ) for( xCoreID = 0; xCoreID < ( BaseType_t ) configNUMBER_OF_CORES; xCoreID++ ) { #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - if( pxCurrentTCBs[ xCoreID ]->xPreemptionDisable == pdFALSE ) + if( pxCurrentTCBs[ xCoreID ]->xPreemptionDisable == 0U ) #endif { if( xYieldPendings[ xCoreID ] != pdFALSE ) From b43c8534760331ab9c084d6926a24e0e0a291527 Mon Sep 17 00:00:00 2001 From: Darian Leung Date: Sun, 16 Jun 2024 00:40:11 +0800 Subject: [PATCH 03/11] feat(freertos/smp): Add granular locking port macros checks Adds the required checks for granular locking port macros. Port Config: - portUSING_GRANULAR_LOCKS to enable granular locks - portCRITICAL_NESTING_IN_TCB should be disabled Granular Locking Port Macros: - Spinlocks - portSPINLOCK_TYPE - portINIT_SPINLOCK( pxSpinlock ) - portINIT_SPINLOCK_STATIC - Locking - portGET_SPINLOCK() - portRELEASE_SPINLOCK() Co-authored-by: Sudeep Mohanty --- include/FreeRTOS.h | 165 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 159 insertions(+), 6 deletions(-) diff --git a/include/FreeRTOS.h b/include/FreeRTOS.h index 1831c8491..5b0adb284 100644 --- a/include/FreeRTOS.h +++ b/include/FreeRTOS.h @@ -369,6 +369,10 @@ #define portCRITICAL_NESTING_IN_TCB 0 #endif +#ifndef portUSING_GRANULAR_LOCKS + #define portUSING_GRANULAR_LOCKS 0 +#endif + #ifndef configMAX_TASK_NAME_LEN #define configMAX_TASK_NAME_LEN 16 #endif @@ -458,7 +462,7 @@ #ifndef portRELEASE_TASK_LOCK - #if ( configNUMBER_OF_CORES == 1 ) + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) ) #define portRELEASE_TASK_LOCK( xCoreID ) #else #error portRELEASE_TASK_LOCK is required in SMP @@ -468,7 +472,7 @@ #ifndef portGET_TASK_LOCK - #if ( configNUMBER_OF_CORES == 1 ) + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) ) #define portGET_TASK_LOCK( xCoreID ) #else #error portGET_TASK_LOCK is required in SMP @@ -478,7 +482,7 @@ #ifndef portRELEASE_ISR_LOCK - #if ( configNUMBER_OF_CORES == 1 ) + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) ) #define portRELEASE_ISR_LOCK( xCoreID ) #else #error portRELEASE_ISR_LOCK is required in SMP @@ -488,7 +492,7 @@ #ifndef portGET_ISR_LOCK - #if ( configNUMBER_OF_CORES == 1 ) + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) ) #define portGET_ISR_LOCK( xCoreID ) #else #error portGET_ISR_LOCK is required in SMP @@ -496,6 +500,30 @@ #endif /* portGET_ISR_LOCK */ +#ifndef portRELEASE_SPINLOCK + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portRELEASE_SPINLOCK is required for granular locking + #endif + +#endif + +#ifndef portGET_SPINLOCK + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portGET_SPINLOCK is required for granular locking + #endif + +#endif + +#ifndef portCHECK_IF_IN_ISR + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portCHECK_IF_IN_ISR is required for granular locking + #endif + +#endif + #ifndef portENTER_CRITICAL_FROM_ISR #if ( configNUMBER_OF_CORES > 1 ) @@ -512,6 +540,126 @@ #endif +#ifndef portENTER_CRITICAL_DATA_GROUP + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portENTER_CRITICAL_DATA_GROUP is required for SMP with granular locking feature enabled + #endif + +#endif + +#ifndef portEXIT_CRITICAL_DATA_GROUP + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portEXIT_CRITICAL_DATA_GROUP is required for SMP with granular locking feature enabled + #endif + +#endif + +#ifndef portENTER_CRITICAL_DATA_GROUP_FROM_ISR + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portENTER_CRITICAL_DATA_GROUP_FROM_ISR is required for SMP with granular locking feature enabled + #endif + +#endif + +#ifndef portEXIT_CRITICAL_DATA_GROUP_FROM_ISR + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portEXIT_CRITICAL_DATA_GROUP_FROM_ISR is required for SMP with granular locking feature enabled + #endif + +#endif + +#ifndef portSPINLOCK_TYPE + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portSPINLOCK_TYPE is required for SMP with granular locking feature enabled + #endif + +#endif + +#ifndef portINIT_EVENT_GROUP_TASK_SPINLOCK + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portINIT_EVENT_GROUP_TASK_SPINLOCK is required for granular locking + #endif + +#endif + +#ifndef portINIT_EVENT_GROUP_ISR_SPINLOCK + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portINIT_EVENT_GROUP_ISR_SPINLOCK is required for granular locking + #endif + +#endif + +#ifndef portINIT_QUEUE_TASK_SPINLOCK + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portINIT_QUEUE_TASK_SPINLOCK is required for granular locking + #endif + +#endif + +#ifndef portINIT_QUEUE_ISR_SPINLOCK + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portINIT_QUEUE_ISR_SPINLOCK is required for granular locking + #endif + +#endif + +#ifndef portINIT_STREAM_BUFFER_TASK_SPINLOCK + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portINIT_STREAM_BUFFER_TASK_SPINLOCK is required for granular locking + #endif + +#endif + +#ifndef portINIT_STREAM_BUFFER_ISR_SPINLOCK + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portINIT_STREAM_BUFFER_ISR_SPINLOCK is required for granular locking + #endif + +#endif + +#ifndef portINIT_KERNEL_TASK_SPINLOCK_STATIC + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portINIT_KERNEL_TASK_SPINLOCK_STATIC is required for granular locking + #endif + +#endif + +#ifndef portINIT_KERNEL_ISR_SPINLOCK_STATIC + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portINIT_KERNEL_ISR_SPINLOCK_STATIC is required for granular locking + #endif + +#endif + +#ifndef portINIT_TIMERS_TASK_SPINLOCK_STATIC + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portINIT_TIMERS_TASK_SPINLOCK_STATIC is required for granular locking + #endif + +#endif + +#ifndef portINIT_TIMERS_ISR_SPINLOCK_STATIC + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #error portINIT_TIMERS_ISR_SPINLOCK_STATIC is required for granular locking + #endif + +#endif + #ifndef configUSE_CORE_AFFINITY #define configUSE_CORE_AFFINITY 0 #endif /* configUSE_CORE_AFFINITY */ @@ -2919,8 +3067,13 @@ /* Either variables of tick type cannot be read atomically, or * portTICK_TYPE_IS_ATOMIC was not set - map the critical sections used when * the tick count is returned to the standard critical section macros. */ - #define portTICK_TYPE_ENTER_CRITICAL() portENTER_CRITICAL() - #define portTICK_TYPE_EXIT_CRITICAL() portEXIT_CRITICAL() + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #define portTICK_TYPE_ENTER_CRITICAL() portENTER_CRITICAL_DATA_GROUP( &xTaskSpinlock, &xISRSpinlock ) + #define portTICK_TYPE_EXIT_CRITICAL() portEXIT_CRITICAL_DATA_GROUP( &xTaskSpinlock, &xISRSpinlock ) + #else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #define portTICK_TYPE_ENTER_CRITICAL() portENTER_CRITICAL() + #define portTICK_TYPE_EXIT_CRITICAL() portEXIT_CRITICAL() + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ #define portTICK_TYPE_SET_INTERRUPT_MASK_FROM_ISR() portSET_INTERRUPT_MASK_FROM_ISR() #define portTICK_TYPE_CLEAR_INTERRUPT_MASK_FROM_ISR( x ) portCLEAR_INTERRUPT_MASK_FROM_ISR( ( x ) ) #else From 212dcfec1b612bb1e5a2faf00a7d3efe13eae153 Mon Sep 17 00:00:00 2001 From: Darian Leung Date: Sun, 16 Jun 2024 00:48:25 +0800 Subject: [PATCH 04/11] feat(granular_locks): Add granular locking functions - Updated prvCheckForRunStateChange() for granular locks - Updated vTaskSuspendAll() and xTaskResumeAll() - Now holds the xTaskSpinlock during kernel suspension - Increments/decrements xPreemptionDisable. Only yields when 0, thus allowing for nested suspensions across different data groups Co-authored-by: Sudeep Mohanty --- include/FreeRTOS.h | 118 +++----------------------- include/task.h | 201 ++++++++++++++++++++++++++++++++++++++++++--- tasks.c | 129 +++++++++++++++++++++-------- 3 files changed, 294 insertions(+), 154 deletions(-) diff --git a/include/FreeRTOS.h b/include/FreeRTOS.h index 5b0adb284..f8a52848f 100644 --- a/include/FreeRTOS.h +++ b/include/FreeRTOS.h @@ -465,7 +465,7 @@ #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) ) #define portRELEASE_TASK_LOCK( xCoreID ) #else - #error portRELEASE_TASK_LOCK is required in SMP + #error portRELEASE_TASK_LOCK is required in SMP without granular locking feature enabled #endif #endif /* portRELEASE_TASK_LOCK */ @@ -475,7 +475,7 @@ #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) ) #define portGET_TASK_LOCK( xCoreID ) #else - #error portGET_TASK_LOCK is required in SMP + #error portGET_TASK_LOCK is required in SMP without granular locking feature enabled #endif #endif /* portGET_TASK_LOCK */ @@ -485,7 +485,7 @@ #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) ) #define portRELEASE_ISR_LOCK( xCoreID ) #else - #error portRELEASE_ISR_LOCK is required in SMP + #error portRELEASE_ISR_LOCK is required in SMP without granular locking feature enabled #endif #endif /* portRELEASE_ISR_LOCK */ @@ -495,7 +495,7 @@ #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) ) #define portGET_ISR_LOCK( xCoreID ) #else - #error portGET_ISR_LOCK is required in SMP + #error portGET_ISR_LOCK is required in SMP without granular locking feature enabled #endif #endif /* portGET_ISR_LOCK */ @@ -503,7 +503,7 @@ #ifndef portRELEASE_SPINLOCK #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portRELEASE_SPINLOCK is required for granular locking + #error portRELEASE_SPINLOCK is required for SMP with granular locking feature enabled #endif #endif @@ -511,7 +511,7 @@ #ifndef portGET_SPINLOCK #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portGET_SPINLOCK is required for granular locking + #error portGET_SPINLOCK is required for SMP with granular locking feature enabled #endif #endif @@ -540,38 +540,6 @@ #endif -#ifndef portENTER_CRITICAL_DATA_GROUP - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portENTER_CRITICAL_DATA_GROUP is required for SMP with granular locking feature enabled - #endif - -#endif - -#ifndef portEXIT_CRITICAL_DATA_GROUP - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portEXIT_CRITICAL_DATA_GROUP is required for SMP with granular locking feature enabled - #endif - -#endif - -#ifndef portENTER_CRITICAL_DATA_GROUP_FROM_ISR - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portENTER_CRITICAL_DATA_GROUP_FROM_ISR is required for SMP with granular locking feature enabled - #endif - -#endif - -#ifndef portEXIT_CRITICAL_DATA_GROUP_FROM_ISR - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portEXIT_CRITICAL_DATA_GROUP_FROM_ISR is required for SMP with granular locking feature enabled - #endif - -#endif - #ifndef portSPINLOCK_TYPE #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) @@ -580,82 +548,18 @@ #endif -#ifndef portINIT_EVENT_GROUP_TASK_SPINLOCK +#ifndef portINIT_SPINLOCK #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_EVENT_GROUP_TASK_SPINLOCK is required for granular locking + #error portINIT_SPINLOCK is required for SMP with granular locking feature enabled #endif #endif -#ifndef portINIT_EVENT_GROUP_ISR_SPINLOCK +#ifndef portINIT_SPINLOCK_STATIC #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_EVENT_GROUP_ISR_SPINLOCK is required for granular locking - #endif - -#endif - -#ifndef portINIT_QUEUE_TASK_SPINLOCK - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_QUEUE_TASK_SPINLOCK is required for granular locking - #endif - -#endif - -#ifndef portINIT_QUEUE_ISR_SPINLOCK - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_QUEUE_ISR_SPINLOCK is required for granular locking - #endif - -#endif - -#ifndef portINIT_STREAM_BUFFER_TASK_SPINLOCK - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_STREAM_BUFFER_TASK_SPINLOCK is required for granular locking - #endif - -#endif - -#ifndef portINIT_STREAM_BUFFER_ISR_SPINLOCK - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_STREAM_BUFFER_ISR_SPINLOCK is required for granular locking - #endif - -#endif - -#ifndef portINIT_KERNEL_TASK_SPINLOCK_STATIC - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_KERNEL_TASK_SPINLOCK_STATIC is required for granular locking - #endif - -#endif - -#ifndef portINIT_KERNEL_ISR_SPINLOCK_STATIC - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_KERNEL_ISR_SPINLOCK_STATIC is required for granular locking - #endif - -#endif - -#ifndef portINIT_TIMERS_TASK_SPINLOCK_STATIC - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_TIMERS_TASK_SPINLOCK_STATIC is required for granular locking - #endif - -#endif - -#ifndef portINIT_TIMERS_ISR_SPINLOCK_STATIC - - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #error portINIT_TIMERS_ISR_SPINLOCK_STATIC is required for granular locking + #error portINIT_SPINLOCK_STATIC is required for SMP with granular locking feature enabled #endif #endif @@ -3076,7 +2980,7 @@ #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ #define portTICK_TYPE_SET_INTERRUPT_MASK_FROM_ISR() portSET_INTERRUPT_MASK_FROM_ISR() #define portTICK_TYPE_CLEAR_INTERRUPT_MASK_FROM_ISR( x ) portCLEAR_INTERRUPT_MASK_FROM_ISR( ( x ) ) -#else +#else /* if ( portTICK_TYPE_IS_ATOMIC == 0 ) */ /* The tick type can be read atomically, so critical sections used when the * tick count is returned can be defined away. */ diff --git a/include/task.h b/include/task.h index 2add58b94..11a1cb251 100644 --- a/include/task.h +++ b/include/task.h @@ -217,7 +217,7 @@ typedef enum * \defgroup taskYIELD taskYIELD * \ingroup SchedulerControl */ -#define taskYIELD() portYIELD() +#define taskYIELD() portYIELD() /** * task. h @@ -231,12 +231,19 @@ typedef enum * \defgroup taskENTER_CRITICAL taskENTER_CRITICAL * \ingroup SchedulerControl */ -#define taskENTER_CRITICAL() portENTER_CRITICAL() +#define taskENTER_CRITICAL() portENTER_CRITICAL() #if ( configNUMBER_OF_CORES == 1 ) - #define taskENTER_CRITICAL_FROM_ISR() portSET_INTERRUPT_MASK_FROM_ISR() + #define taskENTER_CRITICAL_FROM_ISR() portSET_INTERRUPT_MASK_FROM_ISR() #else - #define taskENTER_CRITICAL_FROM_ISR() portENTER_CRITICAL_FROM_ISR() + #define taskENTER_CRITICAL_FROM_ISR() portENTER_CRITICAL_FROM_ISR() #endif +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #define taskLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) portLOCK_DATA_GROUP( ( portSPINLOCK_TYPE * ) pxTaskSpinlock, ( portSPINLOCK_TYPE * ) pxISRSpinlock ) + #define taskLOCK_DATA_GROUP_FROM_ISR( pxISRSpinlock ) portLOCK_DATA_GROUP_FROM_ISR( pxISRSpinlock ) +#else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #define taskLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) taskENTER_CRITICAL() + #define taskLOCK_DATA_GROUP_FROM_ISR( pxISRSpinlock ) taskENTER_CRITICAL_FROM_ISR() +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /** * task. h @@ -250,12 +257,19 @@ typedef enum * \defgroup taskEXIT_CRITICAL taskEXIT_CRITICAL * \ingroup SchedulerControl */ -#define taskEXIT_CRITICAL() portEXIT_CRITICAL() +#define taskEXIT_CRITICAL() portEXIT_CRITICAL() #if ( configNUMBER_OF_CORES == 1 ) - #define taskEXIT_CRITICAL_FROM_ISR( x ) portCLEAR_INTERRUPT_MASK_FROM_ISR( x ) + #define taskEXIT_CRITICAL_FROM_ISR( x ) portCLEAR_INTERRUPT_MASK_FROM_ISR( x ) #else - #define taskEXIT_CRITICAL_FROM_ISR( x ) portEXIT_CRITICAL_FROM_ISR( x ) + #define taskEXIT_CRITICAL_FROM_ISR( x ) portEXIT_CRITICAL_FROM_ISR( x ) #endif +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #define taskUNLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) portUNLOCK_DATA_GROUP( ( portSPINLOCK_TYPE * ) pxTaskSpinlock, ( portSPINLOCK_TYPE * ) pxISRSpinlock ) + #define taskUNLOCK_DATA_GROUP_FROM_ISR( x, pxISRSpinlock ) portUNLOCK_DATA_GROUP_FROM_ISR( x, pxISRSpinlock ) +#else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #define taskUNLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) taskEXIT_CRITICAL() + #define taskUNLOCK_DATA_GROUP_FROM_ISR( x, pxISRSpinlock ) taskEXIT_CRITICAL_FROM_ISR( x ) +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /** * task. h @@ -287,6 +301,146 @@ typedef enum /* Checks if core ID is valid. */ #define taskVALID_CORE_ID( xCoreID ) ( ( ( ( ( BaseType_t ) 0 <= ( xCoreID ) ) && ( ( xCoreID ) < ( BaseType_t ) configNUMBER_OF_CORES ) ) ) ? ( pdTRUE ) : ( pdFALSE ) ) +/** + * task. h + * + * Macro to enter a data group critical section. + * + * \defgroup taskDATA_GROUP_ENTER_CRITICAL taskDATA_GROUP_ENTER_CRITICAL + * \ingroup GranularLocks + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define taskDATA_GROUP_ENTER_CRITICAL( pxTaskSpinlock, pxISRSpinlock ) \ + do { \ + /* Disable preemption to avoid task state changes during the critical section. */ \ + vTaskPreemptionDisable( NULL ); \ + { \ + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ + /* Task spinlock is always taken first */ \ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxTaskSpinlock ); \ + /* Disable interrupts */ \ + portDISABLE_INTERRUPTS(); \ + /* Take the ISR spinlock next */ \ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock ); \ + /* Increment the critical nesting count */ \ + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ + } \ + } while( 0 ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + +/** + * task. h + * + * Macro to enter a data group critical section from an interrupt. + * + * \defgroup taskDATA_GROUP_ENTER_CRITICAL_FROM_ISR taskDATA_GROUP_ENTER_CRITICAL_FROM_ISR + * \ingroup GranularLocks + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define taskDATA_GROUP_ENTER_CRITICAL_FROM_ISR( pxISRSpinlock, puxSavedInterruptStatus ) \ + do { \ + *( puxSavedInterruptStatus ) = portSET_INTERRUPT_MASK_FROM_ISR(); \ + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ + /* Take the ISR spinlock */ \ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock ); \ + /* Increment the critical nesting count */ \ + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ + } while( 0 ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + +/** + * task. h + * + * Macro to exit a data group critical section. + * + * \defgroup taskDATA_GROUP_EXIT_CRITICAL taskDATA_GROUP_EXIT_CRITICAL + * \ingroup GranularLocks + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define taskDATA_GROUP_EXIT_CRITICAL( pxTaskSpinlock, pxISRSpinlock ) \ + do { \ + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ + configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ); \ + /* Release the ISR spinlock */ \ + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock ); \ + /* Release the task spinlock */ \ + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxTaskSpinlock ); \ + /* Decrement the critical nesting count */ \ + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ + /* Enable interrupts only if the critical nesting count is 0 */ \ + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 ) \ + { \ + portENABLE_INTERRUPTS(); \ + } \ + else \ + { \ + mtCOVERAGE_TEST_MARKER(); \ + } \ + /* Re-enable preemption */ \ + vTaskPreemptionEnable( NULL ); \ + } while( 0 ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + +/** + * task. h + * + * Macro to exit a data group critical section from an interrupt. + * + * \defgroup taskDATA_GROUP_EXIT_CRITICAL_FROM_ISR taskDATA_GROUP_EXIT_CRITICAL_FROM_ISR + * \ingroup GranularLocks + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define taskDATA_GROUP_EXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxISRSpinlock ) \ + do { \ + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ + configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ); \ + /* Decrement the critical nesting count */ \ + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ + /* Release the ISR spinlock */ \ + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock ); \ + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 ) \ + { \ + portCLEAR_INTERRUPT_MASK_FROM_ISR( uxSavedInterruptStatus ); \ + } \ + } while( 0 ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + +/** + * task. h + * + * Macros to lock a data group (task-level lock only). + * + * \defgroup taskDATA_GROUP_LOCK taskDATA_GROUP_LOCK + * \ingroup GranularLocks + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define taskDATA_GROUP_LOCK( pxTaskSpinlock ) \ + do { \ + /* Disable preemption while holding the task spinlock. */ \ + vTaskPreemptionDisable( NULL ); \ + { \ + portGET_SPINLOCK( portGET_CORE_ID(), ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ); \ + } \ + } while( 0 ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + +/** + * task. h + * + * Macros to unlock a data group (task-level lock only). + * + * \defgroup taskDATA_GROUP_UNLOCK taskDATA_GROUP_UNLOCK + * \ingroup GranularLocks + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define taskDATA_GROUP_UNLOCK( pxTaskSpinlock ) \ + ( { \ + portRELEASE_SPINLOCK( portGET_CORE_ID(), ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ); \ + /* Re-enable preemption after releasing the task spinlock. */ \ + prvTaskPreemptionEnable( NULL ); \ + } ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + /*----------------------------------------------------------- * TASK CREATION API *----------------------------------------------------------*/ @@ -1488,6 +1642,23 @@ BaseType_t xTaskResumeFromISR( TaskHandle_t xTaskToResume ) PRIVILEGED_FUNCTION; void vTaskPreemptionEnable( const TaskHandle_t xTask ); #endif +#if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + +/* + * THIS FUNCTION MUST NOT BE USED FROM APPLICATION CODE. IT IS ONLY + * INTENDED FOR USE WHEN IMPLEMENTING A PORT OF THE SCHEDULER AND IS + * AN INTERFACE WHICH IS FOR THE EXCLUSIVE USE OF THE SCHEDULER. + * + * @param xTask The handle of the task to enable preemption. Passing NULL + * enables preemption for the calling task. + * + * @return pdTRUE if enabling preemption for the task resulted in a context + * switch, otherwise pdFALSE. This is used by the scheduler to determine if a + * context switch may be required following the enable. + */ + BaseType_t prvTaskPreemptionEnable( const TaskHandle_t xTask ); +#endif + /*----------------------------------------------------------- * SCHEDULER CONTROL *----------------------------------------------------------*/ @@ -3726,7 +3897,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC * It should be used in the implementation of portENTER_CRITICAL if port is running a * multiple core FreeRTOS. */ -#if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) || ( configNUMBER_OF_CORES > 1 ) ) +#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) void vTaskEnterCritical( void ); #endif @@ -3738,7 +3909,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC * It should be used in the implementation of portEXIT_CRITICAL if port is running a * multiple core FreeRTOS. */ -#if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) || ( configNUMBER_OF_CORES > 1 ) ) +#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) void vTaskExitCritical( void ); #endif @@ -3748,7 +3919,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC * should be used in the implementation of portENTER_CRITICAL_FROM_ISR if port is * running a multiple core FreeRTOS. */ -#if ( configNUMBER_OF_CORES > 1 ) +#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) UBaseType_t vTaskEnterCriticalFromISR( void ); #endif @@ -3758,10 +3929,18 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC * should be used in the implementation of portEXIT_CRITICAL_FROM_ISR if port is * running a multiple core FreeRTOS. */ -#if ( configNUMBER_OF_CORES > 1 ) +#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) void vTaskExitCriticalFromISR( UBaseType_t uxSavedInterruptStatus ); #endif +/* + * Checks whether a yield is required after taskUNLOCK_DATA_GROUP() returns. + * To be called while data group is locked. + */ +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + BaseType_t xTaskUnlockCanYield( void ); +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #if ( portUSING_MPU_WRAPPERS == 1 ) /* diff --git a/tasks.c b/tasks.c index 75f03aef7..3310f4758 100644 --- a/tasks.c +++ b/tasks.c @@ -507,7 +507,7 @@ PRIVILEGED_DATA STATIC volatile TickType_t xTickCount = ( TickType_t ) configINI PRIVILEGED_DATA STATIC volatile UBaseType_t uxTopReadyPriority = tskIDLE_PRIORITY; PRIVILEGED_DATA STATIC volatile BaseType_t xSchedulerRunning = pdFALSE; PRIVILEGED_DATA STATIC volatile TickType_t xPendedTicks = ( TickType_t ) 0U; -PRIVILEGED_DATA STATIC volatile BaseType_t xYieldPendings[ configNUMBER_OF_CORES ] = { pdFALSE }; +PRIVILEGED_DATA volatile BaseType_t xYieldPendings[ configNUMBER_OF_CORES ] = { pdFALSE }; PRIVILEGED_DATA STATIC volatile BaseType_t xNumOfOverflows = ( BaseType_t ) 0; PRIVILEGED_DATA STATIC UBaseType_t uxTaskNumber = ( UBaseType_t ) 0U; PRIVILEGED_DATA STATIC volatile TickType_t xNextTaskUnblockTime = ( TickType_t ) 0U; /* Initialised to portMAX_DELAY before the scheduler starts. */ @@ -541,6 +541,11 @@ PRIVILEGED_DATA STATIC volatile configRUN_TIME_COUNTER_TYPE ulTotalRunTime[ conf #endif +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + PRIVILEGED_DATA static portSPINLOCK_TYPE xTaskSpinlock = portINIT_KERNEL_TASK_SPINLOCK_STATIC; + PRIVILEGED_DATA static portSPINLOCK_TYPE xISRSpinlock = portINIT_KERNEL_ISR_SPINLOCK_STATIC; +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + /*-----------------------------------------------------------*/ /* File private functions. --------------------------------*/ @@ -550,14 +555,14 @@ PRIVILEGED_DATA STATIC volatile configRUN_TIME_COUNTER_TYPE ulTotalRunTime[ conf */ STATIC BaseType_t prvCreateIdleTasks( void ); -#if ( configNUMBER_OF_CORES > 1 ) +#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) /* * Checks to see if another task moved the current task out of the ready * list while it was waiting to enter a critical section and yields, if so. */ STATIC void prvCheckForRunStateChange( void ); -#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ #if ( configNUMBER_OF_CORES > 1 ) @@ -823,7 +828,7 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; #endif /* #if ( ( configUSE_TRACE_FACILITY == 1 ) && ( configUSE_STATS_FORMATTING_FUNCTIONS > 0 ) ) */ /*-----------------------------------------------------------*/ -#if ( configNUMBER_OF_CORES > 1 ) +#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) STATIC void prvCheckForRunStateChange( void ) { UBaseType_t uxPrevCriticalNesting; @@ -887,7 +892,7 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; } } } -#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /*-----------------------------------------------------------*/ @@ -3925,31 +3930,45 @@ void vTaskSuspendAll( void ) * do not otherwise exhibit real time behaviour. */ portSOFTWARE_BARRIER(); - portGET_TASK_LOCK( xCoreID ); - - /* uxSchedulerSuspended is increased after prvCheckForRunStateChange. The - * purpose is to prevent altering the variable when fromISR APIs are readying - * it. */ - if( uxSchedulerSuspended == 0U ) + #if ( portUSING_GRANULAR_LOCKS == 1 ) { - prvCheckForRunStateChange(); + portGET_SPINLOCK( &xTaskSpinlock ); + portGET_SPINLOCK( &xISRSpinlock ); + + /* Increment xPreemptionDisable to prevent preemption and also + * track whether to yield in xTaskResumeAll(). */ + pxCurrentTCBs[ portGET_CORE_ID() ]->xPreemptionDisable++; + + /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment + * is used to allow calls to vTaskSuspendAll() to nest. */ + ++uxSchedulerSuspended; + + portRELEASE_SPINLOCK( &xISRSpinlock ); } - else + #else /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ { - mtCOVERAGE_TEST_MARKER(); + portGET_TASK_LOCK(); + + /* uxSchedulerSuspended is increased after prvCheckForRunStateChange. The + * purpose is to prevent altering the variable when fromISR APIs are readying + * it. */ + if( uxSchedulerSuspended == 0U ) + { + prvCheckForRunStateChange(); + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + + portGET_ISR_LOCK(); + + /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment + * is used to allow calls to vTaskSuspendAll() to nest. */ + ++uxSchedulerSuspended; + portRELEASE_ISR_LOCK(); } - - /* Query the coreID again as prvCheckForRunStateChange may have - * caused the task to get scheduled on a different core. The correct - * task lock for the core is acquired in prvCheckForRunStateChange. */ - xCoreID = ( BaseType_t ) portGET_CORE_ID(); - - portGET_ISR_LOCK( xCoreID ); - - /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment - * is used to allow calls to vTaskSuspendAll() to nest. */ - ++uxSchedulerSuspended; - portRELEASE_ISR_LOCK( xCoreID ); + #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ portCLEAR_INTERRUPT_MASK( ulState ); } @@ -4054,7 +4073,24 @@ BaseType_t xTaskResumeAll( void ) configASSERT( uxSchedulerSuspended != 0U ); uxSchedulerSuspended = ( UBaseType_t ) ( uxSchedulerSuspended - 1U ); - portRELEASE_TASK_LOCK( xCoreID ); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + { + configASSERT( pxCurrentTCBs[ portGET_CORE_ID() ]->xPreemptionDisable > 0 ); + + /* Decrement xPreemptionDisable. If 0, it means this we are not + * in a nested suspension scenario, thus this function and yield + * if necessary. */ + pxCurrentTCBs[ portGET_CORE_ID() ]->xPreemptionDisable--; + + /* Release the kernel's task spinlock that was held throughout + * the kernel suspension. */ + portRELEASE_SPINLOCK( &xTaskSpinlock ); + } + #else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + { + portRELEASE_TASK_LOCK(); + } + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ if( uxSchedulerSuspended == ( UBaseType_t ) 0U ) { @@ -7045,7 +7081,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) #endif /* #if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) && ( configNUMBER_OF_CORES == 1 ) ) */ /*-----------------------------------------------------------*/ -#if ( configNUMBER_OF_CORES > 1 ) +#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) void vTaskEnterCritical( void ) { @@ -7094,11 +7130,10 @@ STATIC void prvResetNextTaskUnblockTime( void ) traceRETURN_vTaskEnterCritical(); } -#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ - +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /*-----------------------------------------------------------*/ -#if ( configNUMBER_OF_CORES > 1 ) +#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) UBaseType_t vTaskEnterCriticalFromISR( void ) { @@ -7128,7 +7163,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) return uxSavedInterruptStatus; } -#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /*-----------------------------------------------------------*/ #if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) && ( configNUMBER_OF_CORES == 1 ) ) @@ -7176,7 +7211,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) #endif /* #if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) && ( configNUMBER_OF_CORES == 1 ) ) */ /*-----------------------------------------------------------*/ -#if ( configNUMBER_OF_CORES > 1 ) +#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) void vTaskExitCritical( void ) { @@ -7236,10 +7271,10 @@ STATIC void prvResetNextTaskUnblockTime( void ) traceRETURN_vTaskExitCritical(); } -#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /*-----------------------------------------------------------*/ -#if ( configNUMBER_OF_CORES > 1 ) +#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) void vTaskExitCriticalFromISR( UBaseType_t uxSavedInterruptStatus ) { @@ -7282,7 +7317,29 @@ STATIC void prvResetNextTaskUnblockTime( void ) traceRETURN_vTaskExitCriticalFromISR(); } -#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ +/*-----------------------------------------------------------*/ + +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + + BaseType_t xTaskUnlockCanYield( void ) + { + BaseType_t xReturn; + BaseType_t xCoreID = portGET_CORE_ID(); + + if( ( xYieldPendings[ xCoreID ] == pdTRUE ) && ( uxSchedulerSuspended == pdFALSE ) && ( pxCurrentTCBs[ xCoreID ]->xPreemptionDisable == pdFALSE ) ) + { + xReturn = pdTRUE; + } + else + { + xReturn = pdFALSE; + } + + return xReturn; + } + +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /*-----------------------------------------------------------*/ #if ( configUSE_STATS_FORMATTING_FUNCTIONS > 0 ) From 62f334a51e4325d7bcdfa06776510cd8ba731c90 Mon Sep 17 00:00:00 2001 From: Darian Leung Date: Sun, 16 Jun 2024 00:50:15 +0800 Subject: [PATCH 05/11] change(freertos/smp): Update tasks.c locking Updated critical section macros with granular locks. Some tasks.c API relied on their callers to enter critical sections. This assumption no longer works under granular locking. Critical sections added to the following functions: - `vTaskInternalSetTimeOutState()` - `xTaskIncrementTick()` - `vTaskSwitchContext()` - `xTaskRemoveFromEventList()` - `vTaskInternalSetTimeOutState()` - `eTaskConfirmSleepModeStatus()` - `xTaskPriorityDisinherit()` - `pvTaskIncrementMutexHeldCount()` Added missing suspensions to the following functions: - `vTaskPlaceOnEventList()` - `vTaskPlaceOnUnorderedEventList()` - `vTaskPlaceOnEventListRestricted()` Fixed the locking in vTaskSwitchContext() vTaskSwitchContext() must aquire both kernel locks, viz., task lock and ISR lock. This is because, vTaskSwitchContext() can be called from either task context or ISR context. Also, vTaskSwitchContext() must not alter the interrupt state prematurely. Co-authored-by: Sudeep Mohanty --- include/FreeRTOS.h | 7 +- include/task.h | 98 ++- tasks.c | 1622 ++++++++++++++++++++++++++++---------------- 3 files changed, 1093 insertions(+), 634 deletions(-) diff --git a/include/FreeRTOS.h b/include/FreeRTOS.h index f8a52848f..e3f3bb300 100644 --- a/include/FreeRTOS.h +++ b/include/FreeRTOS.h @@ -2972,8 +2972,8 @@ * portTICK_TYPE_IS_ATOMIC was not set - map the critical sections used when * the tick count is returned to the standard critical section macros. */ #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #define portTICK_TYPE_ENTER_CRITICAL() portENTER_CRITICAL_DATA_GROUP( &xTaskSpinlock, &xISRSpinlock ) - #define portTICK_TYPE_EXIT_CRITICAL() portEXIT_CRITICAL_DATA_GROUP( &xTaskSpinlock, &xISRSpinlock ) + #define portTICK_TYPE_ENTER_CRITICAL() kernelENTER_CRITICAL() + #define portTICK_TYPE_EXIT_CRITICAL() kernelEXIT_CRITICAL() #else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ #define portTICK_TYPE_ENTER_CRITICAL() portENTER_CRITICAL() #define portTICK_TYPE_EXIT_CRITICAL() portEXIT_CRITICAL() @@ -3242,6 +3242,9 @@ typedef struct xSTATIC_TCB #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) UBaseType_t xDummy25; #endif + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + BaseType_t xDummy26; + #endif #if ( ( portSTACK_GROWTH > 0 ) || ( configRECORD_STACK_HIGH_ADDRESS == 1 ) ) void * pxDummy8; #endif diff --git a/include/task.h b/include/task.h index 11a1cb251..57da38118 100644 --- a/include/task.h +++ b/include/task.h @@ -217,7 +217,7 @@ typedef enum * \defgroup taskYIELD taskYIELD * \ingroup SchedulerControl */ -#define taskYIELD() portYIELD() +#define taskYIELD() portYIELD() /** * task. h @@ -231,19 +231,12 @@ typedef enum * \defgroup taskENTER_CRITICAL taskENTER_CRITICAL * \ingroup SchedulerControl */ -#define taskENTER_CRITICAL() portENTER_CRITICAL() +#define taskENTER_CRITICAL() portENTER_CRITICAL() #if ( configNUMBER_OF_CORES == 1 ) - #define taskENTER_CRITICAL_FROM_ISR() portSET_INTERRUPT_MASK_FROM_ISR() + #define taskENTER_CRITICAL_FROM_ISR() portSET_INTERRUPT_MASK_FROM_ISR() #else - #define taskENTER_CRITICAL_FROM_ISR() portENTER_CRITICAL_FROM_ISR() + #define taskENTER_CRITICAL_FROM_ISR() portENTER_CRITICAL_FROM_ISR() #endif -#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #define taskLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) portLOCK_DATA_GROUP( ( portSPINLOCK_TYPE * ) pxTaskSpinlock, ( portSPINLOCK_TYPE * ) pxISRSpinlock ) - #define taskLOCK_DATA_GROUP_FROM_ISR( pxISRSpinlock ) portLOCK_DATA_GROUP_FROM_ISR( pxISRSpinlock ) -#else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ - #define taskLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) taskENTER_CRITICAL() - #define taskLOCK_DATA_GROUP_FROM_ISR( pxISRSpinlock ) taskENTER_CRITICAL_FROM_ISR() -#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /** * task. h @@ -257,19 +250,12 @@ typedef enum * \defgroup taskEXIT_CRITICAL taskEXIT_CRITICAL * \ingroup SchedulerControl */ -#define taskEXIT_CRITICAL() portEXIT_CRITICAL() +#define taskEXIT_CRITICAL() portEXIT_CRITICAL() #if ( configNUMBER_OF_CORES == 1 ) - #define taskEXIT_CRITICAL_FROM_ISR( x ) portCLEAR_INTERRUPT_MASK_FROM_ISR( x ) + #define taskEXIT_CRITICAL_FROM_ISR( x ) portCLEAR_INTERRUPT_MASK_FROM_ISR( x ) #else - #define taskEXIT_CRITICAL_FROM_ISR( x ) portEXIT_CRITICAL_FROM_ISR( x ) + #define taskEXIT_CRITICAL_FROM_ISR( x ) portEXIT_CRITICAL_FROM_ISR( x ) #endif -#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - #define taskUNLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) portUNLOCK_DATA_GROUP( ( portSPINLOCK_TYPE * ) pxTaskSpinlock, ( portSPINLOCK_TYPE * ) pxISRSpinlock ) - #define taskUNLOCK_DATA_GROUP_FROM_ISR( x, pxISRSpinlock ) portUNLOCK_DATA_GROUP_FROM_ISR( x, pxISRSpinlock ) -#else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ - #define taskUNLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) taskEXIT_CRITICAL() - #define taskUNLOCK_DATA_GROUP_FROM_ISR( x, pxISRSpinlock ) taskEXIT_CRITICAL_FROM_ISR( x ) -#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /** * task. h @@ -317,11 +303,11 @@ typedef enum { \ const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ /* Task spinlock is always taken first */ \ - portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxTaskSpinlock ); \ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ); \ /* Disable interrupts */ \ portDISABLE_INTERRUPTS(); \ /* Take the ISR spinlock next */ \ - portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock ); \ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) ( pxISRSpinlock ) ); \ /* Increment the critical nesting count */ \ portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ } \ @@ -340,11 +326,13 @@ typedef enum #define taskDATA_GROUP_ENTER_CRITICAL_FROM_ISR( pxISRSpinlock, puxSavedInterruptStatus ) \ do { \ *( puxSavedInterruptStatus ) = portSET_INTERRUPT_MASK_FROM_ISR(); \ - const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ - /* Take the ISR spinlock */ \ - portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock ); \ - /* Increment the critical nesting count */ \ - portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ + { \ + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ + /* Take the ISR spinlock */ \ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) ( pxISRSpinlock ) ); \ + /* Increment the critical nesting count */ \ + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ + } \ } while( 0 ) #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ @@ -357,27 +345,27 @@ typedef enum * \ingroup GranularLocks */ #if ( portUSING_GRANULAR_LOCKS == 1 ) - #define taskDATA_GROUP_EXIT_CRITICAL( pxTaskSpinlock, pxISRSpinlock ) \ - do { \ - const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ - configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ); \ - /* Release the ISR spinlock */ \ - portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock ); \ - /* Release the task spinlock */ \ - portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxTaskSpinlock ); \ - /* Decrement the critical nesting count */ \ - portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ - /* Enable interrupts only if the critical nesting count is 0 */ \ - if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 ) \ - { \ - portENABLE_INTERRUPTS(); \ - } \ - else \ - { \ - mtCOVERAGE_TEST_MARKER(); \ - } \ - /* Re-enable preemption */ \ - vTaskPreemptionEnable( NULL ); \ + #define taskDATA_GROUP_EXIT_CRITICAL( pxTaskSpinlock, pxISRSpinlock ) \ + do { \ + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ + configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ); \ + /* Release the ISR spinlock */ \ + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) ( pxISRSpinlock ) ); \ + /* Release the task spinlock */ \ + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ); \ + /* Decrement the critical nesting count */ \ + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ + /* Enable interrupts only if the critical nesting count is 0 */ \ + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 ) \ + { \ + portENABLE_INTERRUPTS(); \ + } \ + else \ + { \ + mtCOVERAGE_TEST_MARKER(); \ + } \ + /* Re-enable preemption */ \ + prvTaskPreemptionEnable( NULL ); \ } while( 0 ) #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ @@ -397,7 +385,7 @@ typedef enum /* Decrement the critical nesting count */ \ portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ /* Release the ISR spinlock */ \ - portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock ); \ + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) ( pxISRSpinlock ) ); \ if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 ) \ { \ portCLEAR_INTERRUPT_MASK_FROM_ISR( uxSavedInterruptStatus ); \ @@ -3897,7 +3885,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC * It should be used in the implementation of portENTER_CRITICAL if port is running a * multiple core FreeRTOS. */ -#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) +#if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) || ( configNUMBER_OF_CORES > 1 ) ) void vTaskEnterCritical( void ); #endif @@ -3909,7 +3897,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC * It should be used in the implementation of portEXIT_CRITICAL if port is running a * multiple core FreeRTOS. */ -#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) +#if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) || ( configNUMBER_OF_CORES > 1 ) ) void vTaskExitCritical( void ); #endif @@ -3919,7 +3907,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC * should be used in the implementation of portENTER_CRITICAL_FROM_ISR if port is * running a multiple core FreeRTOS. */ -#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) +#if ( configNUMBER_OF_CORES > 1 ) UBaseType_t vTaskEnterCriticalFromISR( void ); #endif @@ -3929,12 +3917,12 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC * should be used in the implementation of portEXIT_CRITICAL_FROM_ISR if port is * running a multiple core FreeRTOS. */ -#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) +#if ( configNUMBER_OF_CORES > 1 ) void vTaskExitCriticalFromISR( UBaseType_t uxSavedInterruptStatus ); #endif /* - * Checks whether a yield is required after taskUNLOCK_DATA_GROUP() returns. + * Checks whether a yield is required after portUNLOCK_DATA_GROUP() returns. * To be called while data group is locked. */ #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) diff --git a/tasks.c b/tasks.c index 3310f4758..8401f2cef 100644 --- a/tasks.c +++ b/tasks.c @@ -137,11 +137,15 @@ /* * Macros used by vListTask to indicate which state a task is in. */ -#define tskRUNNING_CHAR ( 'X' ) -#define tskBLOCKED_CHAR ( 'B' ) -#define tskREADY_CHAR ( 'R' ) -#define tskDELETED_CHAR ( 'D' ) -#define tskSUSPENDED_CHAR ( 'S' ) +#define tskRUNNING_CHAR ( 'X' ) +#define tskBLOCKED_CHAR ( 'B' ) +#define tskREADY_CHAR ( 'R' ) +#define tskDELETED_CHAR ( 'D' ) +#define tskSUSPENDED_CHAR ( 'S' ) + +/* Bits used to record a deferred state change of a task. */ +#define tskDEFERRED_DELETION ( UBaseType_t ) ( 1U << 0U ) +#define tskDEFERRED_SUSPENSION ( UBaseType_t ) ( 1U << 1U ) /* * Some kernel aware debuggers require the data the debugger needs access to be @@ -349,7 +353,33 @@ /* Yields the given core. This must be called from a critical section and xCoreID * must be valid. This macro is not required in single core since there is only * one core to yield. */ - #define prvYieldCore( xCoreID ) \ + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + #define prvYieldCore( xCoreID ) \ + do { \ + if( ( xCoreID ) == ( BaseType_t ) portGET_CORE_ID() ) \ + { \ + /* Pending a yield for this core since it is in the critical section. */ \ + xYieldPendings[ ( xCoreID ) ] = pdTRUE; \ + } \ + else \ + { \ + if( pxCurrentTCBs[ ( xCoreID ) ]->uxPreemptionDisable == 0U ) \ + { \ + /* Request other core to yield if it is not requested before. */ \ + if( pxCurrentTCBs[ ( xCoreID ) ]->xTaskRunState != taskTASK_SCHEDULED_TO_YIELD ) \ + { \ + portYIELD_CORE( xCoreID ); \ + pxCurrentTCBs[ ( xCoreID ) ]->xTaskRunState = taskTASK_SCHEDULED_TO_YIELD; \ + } \ + } \ + else \ + { \ + xYieldPendings[ ( xCoreID ) ] = pdTRUE; \ + } \ + } \ + } while( 0 ) + #else /* if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + #define prvYieldCore( xCoreID ) \ do { \ if( ( xCoreID ) == ( BaseType_t ) portGET_CORE_ID() ) \ { \ @@ -366,9 +396,42 @@ } \ } \ } while( 0 ) + #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ #endif /* #if ( configNUMBER_OF_CORES > 1 ) */ /*-----------------------------------------------------------*/ +/* Macros to take and release kernel spinlocks */ +#if ( configNUMBER_OF_CORES > 1 ) + #if ( portUSING_GRANULAR_LOCKS == 1 ) + #define kernelGET_TASK_LOCK( xCoreID ) portGET_SPINLOCK( xCoreID, &xTaskSpinlock ) + #define kernelRELEASE_TASK_LOCK( xCoreID ) portRELEASE_SPINLOCK( xCoreID, &xTaskSpinlock ) + #define kernelGET_ISR_LOCK( xCoreID ) portGET_SPINLOCK( xCoreID, &xISRSpinlock ) + #define kernelRELEASE_ISR_LOCK( xCoreID ) portRELEASE_SPINLOCK( xCoreID, &xISRSpinlock ) + #else + #define kernelGET_TASK_LOCK( xCoreID ) portGET_TASK_LOCK( xCoreID ) + #define kernelRELEASE_TASK_LOCK( xCoreID ) portRELEASE_TASK_LOCK( xCoreID ) + #define kernelGET_ISR_LOCK( xCoreID ) portGET_ISR_LOCK( xCoreID ) + #define kernelRELEASE_ISR_LOCK( xCoreID ) portRELEASE_ISR_LOCK( xCoreID ) + #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ +#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ + +/* + * Macros to mark the start and end of a critical code region. + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define kernelENTER_CRITICAL() vTaskEnterCritical() + #define kernelENTER_CRITICAL_FROM_ISR() vTaskEnterCriticalFromISR() + #define kernelEXIT_CRITICAL() vTaskExitCritical() + #define kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) vTaskExitCriticalFromISR( uxSavedInterruptStatus ) +#else /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + #define kernelENTER_CRITICAL() taskENTER_CRITICAL() + #define kernelENTER_CRITICAL_FROM_ISR() taskENTER_CRITICAL_FROM_ISR() + #define kernelEXIT_CRITICAL() taskEXIT_CRITICAL() + #define kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + +/*-----------------------------------------------------------*/ + /* * Task control block. A task control block (TCB) is allocated for each task, * and stores task state information, including a pointer to the task's context @@ -397,7 +460,11 @@ typedef struct tskTaskControlBlock /* The old naming convention is used to char pcTaskName[ configMAX_TASK_NAME_LEN ]; /**< Descriptive name given to the task when created. Facilitates debugging only. */ #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - UBaseType_t xPreemptionDisable; /**< Used to prevent the task from being preempted. */ + UBaseType_t uxPreemptionDisable; /**< Used to prevent the task from being preempted. */ + #endif + + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + UBaseType_t uxDeferredStateChange; /**< Used to indicate if the task's state change is deferred. */ #endif #if ( ( portSTACK_GROWTH > 0 ) || ( configRECORD_STACK_HIGH_ADDRESS == 1 ) ) @@ -541,9 +608,10 @@ PRIVILEGED_DATA STATIC volatile configRUN_TIME_COUNTER_TYPE ulTotalRunTime[ conf #endif +/* Kernel spinlock variables when using granular locking */ #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - PRIVILEGED_DATA static portSPINLOCK_TYPE xTaskSpinlock = portINIT_KERNEL_TASK_SPINLOCK_STATIC; - PRIVILEGED_DATA static portSPINLOCK_TYPE xISRSpinlock = portINIT_KERNEL_ISR_SPINLOCK_STATIC; + PRIVILEGED_DATA static portSPINLOCK_TYPE xTaskSpinlock = portINIT_SPINLOCK_STATIC; + PRIVILEGED_DATA static portSPINLOCK_TYPE xISRSpinlock = portINIT_SPINLOCK_STATIC; #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /*-----------------------------------------------------------*/ @@ -555,14 +623,14 @@ PRIVILEGED_DATA STATIC volatile configRUN_TIME_COUNTER_TYPE ulTotalRunTime[ conf */ STATIC BaseType_t prvCreateIdleTasks( void ); -#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) +#if ( configNUMBER_OF_CORES > 1 ) /* * Checks to see if another task moved the current task out of the ready * list while it was waiting to enter a critical section and yields, if so. */ STATIC void prvCheckForRunStateChange( void ); -#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ +#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ #if ( configNUMBER_OF_CORES > 1 ) @@ -785,6 +853,13 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; TaskHandle_t * const pxCreatedTask ) PRIVILEGED_FUNCTION; #endif /* #if ( configSUPPORT_DYNAMIC_ALLOCATION == 1 ) */ +/* + * Helper function to enable preemption for a task. + */ +#if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + BaseType_t prvTaskPreemptionEnable( const TaskHandle_t xTask ) PRIVILEGED_FUNCTION; +#endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + /* * freertos_tasks_c_additions_init() should only be called if the user definable * macro FREERTOS_TASKS_C_ADDITIONS_INIT() is defined, as that is the only macro @@ -828,10 +903,9 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; #endif /* #if ( ( configUSE_TRACE_FACILITY == 1 ) && ( configUSE_STATS_FORMATTING_FUNCTIONS > 0 ) ) */ /*-----------------------------------------------------------*/ -#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) +#if ( configNUMBER_OF_CORES > 1 ) STATIC void prvCheckForRunStateChange( void ) { - UBaseType_t uxPrevCriticalNesting; const TCB_t * pxThisTCB; BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); @@ -844,6 +918,8 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; while( pxThisTCB->xTaskRunState == taskTASK_SCHEDULED_TO_YIELD ) { + UBaseType_t uxPrevCriticalNesting; + /* We are only here if we just entered a critical section * or if we just suspended the scheduler, and another task * has requested that we yield. @@ -857,7 +933,7 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; if( uxPrevCriticalNesting > 0U ) { portSET_CRITICAL_NESTING_COUNT( xCoreID, 0U ); - portRELEASE_ISR_LOCK( xCoreID ); + kernelRELEASE_ISR_LOCK( xCoreID ); } else { @@ -866,7 +942,7 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; mtCOVERAGE_TEST_MARKER(); } - portRELEASE_TASK_LOCK( xCoreID ); + kernelRELEASE_TASK_LOCK( xCoreID ); portMEMORY_BARRIER(); configASSERT( pxThisTCB->xTaskRunState == taskTASK_SCHEDULED_TO_YIELD ); @@ -881,18 +957,18 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; portDISABLE_INTERRUPTS(); xCoreID = ( BaseType_t ) portGET_CORE_ID(); - portGET_TASK_LOCK( xCoreID ); - portGET_ISR_LOCK( xCoreID ); + kernelGET_TASK_LOCK( xCoreID ); + kernelGET_ISR_LOCK( xCoreID ); portSET_CRITICAL_NESTING_COUNT( xCoreID, uxPrevCriticalNesting ); if( uxPrevCriticalNesting == 0U ) { - portRELEASE_ISR_LOCK( xCoreID ); + kernelRELEASE_ISR_LOCK( xCoreID ); } } } -#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ +#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ /*-----------------------------------------------------------*/ @@ -951,12 +1027,23 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; #endif { #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - if( pxCurrentTCBs[ xCoreID ]->xPreemptionDisable == 0U ) - #endif + { + if( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) + { + xLowestPriorityToPreempt = xCurrentCoreTaskPriority; + xLowestPriorityCore = xCoreID; + } + else + { + xYieldPendings[ xCoreID ] = pdTRUE; + } + } + #else /* if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ { xLowestPriorityToPreempt = xCurrentCoreTaskPriority; xLowestPriorityCore = xCoreID; } + #endif /* if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ } } else @@ -1257,12 +1344,23 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; ( xYieldPendings[ uxCore ] == pdFALSE ) ) { #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - if( pxCurrentTCBs[ uxCore ]->xPreemptionDisable == 0U ) - #endif + { + if( pxCurrentTCBs[ uxCore ]->uxPreemptionDisable == 0U ) + { + xLowestPriority = xTaskPriority; + xLowestPriorityCore = ( BaseType_t ) uxCore; + } + else + { + xYieldPendings[ uxCore ] = pdTRUE; + } + } + #else /* if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ { xLowestPriority = xTaskPriority; xLowestPriorityCore = ( BaseType_t ) uxCore; } + #endif /* if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ } } } @@ -2062,7 +2160,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, { /* Ensure interrupts don't access the task lists while the lists are being * updated. */ - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { uxCurrentNumberOfTasks = ( UBaseType_t ) ( uxCurrentNumberOfTasks + 1U ); @@ -2120,7 +2218,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, portSETUP_TCB( pxNewTCB ); } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); if( xSchedulerRunning != pdFALSE ) { @@ -2140,7 +2238,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, { /* Ensure interrupts don't access the task lists while the lists are being * updated. */ - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { uxCurrentNumberOfTasks++; @@ -2189,7 +2287,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, mtCOVERAGE_TEST_MARKER(); } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); } #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ @@ -2236,144 +2334,173 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, BaseType_t xDeleteTCBInIdleTask = pdFALSE; BaseType_t xTaskIsRunningOrYielding; + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + BaseType_t xDeferredDeletion = pdFALSE; + #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + traceENTER_vTaskDelete( xTaskToDelete ); - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { /* If null is passed in here then it is the calling task that is * being deleted. */ pxTCB = prvGetTCBFromHandle( xTaskToDelete ); configASSERT( pxTCB != NULL ); - /* Remove task from the ready/delayed list. */ - if( uxListRemove( &( pxTCB->xStateListItem ) ) == ( UBaseType_t ) 0 ) - { - taskRESET_READY_PRIORITY( pxTCB->uxPriority ); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - /* Is the task waiting on an event also? */ - if( listLIST_ITEM_CONTAINER( &( pxTCB->xEventListItem ) ) != NULL ) - { - ( void ) uxListRemove( &( pxTCB->xEventListItem ) ); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } - - /* Increment the uxTaskNumber also so kernel aware debuggers can - * detect that the task lists need re-generating. This is done before - * portPRE_TASK_DELETE_HOOK() as in the Windows port that macro will - * not return. */ - uxTaskNumber++; - - /* Use temp variable as distinct sequence points for reading volatile - * variables prior to a logical operator to ensure compliance with - * MISRA C 2012 Rule 13.5. */ - xTaskIsRunningOrYielding = taskTASK_IS_RUNNING_OR_SCHEDULED_TO_YIELD( pxTCB ); - - /* If the task is running (or yielding), we must add it to the - * termination list so that an idle task can delete it when it is - * no longer running. */ - if( ( xSchedulerRunning != pdFALSE ) && ( xTaskIsRunningOrYielding != pdFALSE ) ) - { - /* A running task or a task which is scheduled to yield is being - * deleted. This cannot complete when the task is still running - * on a core, as a context switch to another task is required. - * Place the task in the termination list. The idle task will check - * the termination list and free up any memory allocated by the - * scheduler for the TCB and stack of the deleted task. */ - vListInsertEnd( &xTasksWaitingTermination, &( pxTCB->xStateListItem ) ); - - /* Increment the ucTasksDeleted variable so the idle task knows - * there is a task that has been deleted and that it should therefore - * check the xTasksWaitingTermination list. */ - ++uxDeletedTasksWaitingCleanUp; - - /* Call the delete hook before portPRE_TASK_DELETE_HOOK() as - * portPRE_TASK_DELETE_HOOK() does not return in the Win32 port. */ - traceTASK_DELETE( pxTCB ); - - /* Delete the task TCB in idle task. */ - xDeleteTCBInIdleTask = pdTRUE; - - /* The pre-delete hook is primarily for the Windows simulator, - * in which Windows specific clean up operations are performed, - * after which it is not possible to yield away from this task - - * hence xYieldPending is used to latch that a context switch is - * required. */ - #if ( configNUMBER_OF_CORES == 1 ) - portPRE_TASK_DELETE_HOOK( pxTCB, &( xYieldPendings[ 0 ] ) ); - #else - portPRE_TASK_DELETE_HOOK( pxTCB, &( xYieldPendings[ pxTCB->xTaskRunState ] ) ); - #endif - - /* In the case of SMP, it is possible that the task being deleted - * is running on another core. We must evict the task before - * exiting the critical section to ensure that the task cannot - * take an action which puts it back on ready/state/event list, - * thereby nullifying the delete operation. Once evicted, the - * task won't be scheduled ever as it will no longer be on the - * ready list. */ - #if ( configNUMBER_OF_CORES > 1 ) + /* If the task has disabled preemption, we need to defer the deletion until the + * task enables preemption. The deletion will be performed in vTaskPreemptionEnable(). */ + if( pxTCB->uxPreemptionDisable > 0U ) { - if( taskTASK_IS_RUNNING( pxTCB ) == pdTRUE ) - { - if( pxTCB->xTaskRunState == ( BaseType_t ) portGET_CORE_ID() ) - { - configASSERT( uxSchedulerSuspended == 0 ); - taskYIELD_WITHIN_API(); - } - else - { - prvYieldCore( pxTCB->xTaskRunState ); - } - } - } - #endif /* #if ( configNUMBER_OF_CORES > 1 ) */ - } - else - { - --uxCurrentNumberOfTasks; - traceTASK_DELETE( pxTCB ); - - /* Reset the next expected unblock time in case it referred to - * the task that has just been deleted. */ - prvResetNextTaskUnblockTime(); - } - } - taskEXIT_CRITICAL(); - - /* If the task is not deleting itself, call prvDeleteTCB from outside of - * critical section. If a task deletes itself, prvDeleteTCB is called - * from prvCheckTasksWaitingTermination which is called from Idle task. */ - if( xDeleteTCBInIdleTask != pdTRUE ) - { - prvDeleteTCB( pxTCB ); - } - - /* Force a reschedule if it is the currently running task that has just - * been deleted. */ - #if ( configNUMBER_OF_CORES == 1 ) - { - if( xSchedulerRunning != pdFALSE ) - { - if( pxTCB == pxCurrentTCB ) - { - configASSERT( uxSchedulerSuspended == 0 ); - taskYIELD_WITHIN_API(); + pxTCB->uxDeferredStateChange |= tskDEFERRED_DELETION; + xDeferredDeletion = pdTRUE; } else { mtCOVERAGE_TEST_MARKER(); } + #endif /* configUSE_TASK_PREEMPTION_DISABLE */ + + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + if( xDeferredDeletion == pdFALSE ) + #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + { + /* Remove task from the ready/delayed list. */ + if( uxListRemove( &( pxTCB->xStateListItem ) ) == ( UBaseType_t ) 0 ) + { + taskRESET_READY_PRIORITY( pxTCB->uxPriority ); + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + + /* Is the task waiting on an event also? */ + if( listLIST_ITEM_CONTAINER( &( pxTCB->xEventListItem ) ) != NULL ) + { + ( void ) uxListRemove( &( pxTCB->xEventListItem ) ); + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + + /* Increment the uxTaskNumber also so kernel aware debuggers can + * detect that the task lists need re-generating. This is done before + * portPRE_TASK_DELETE_HOOK() as in the Windows port that macro will + * not return. */ + uxTaskNumber++; + + /* Use temp variable as distinct sequence points for reading volatile + * variables prior to a logical operator to ensure compliance with + * MISRA C 2012 Rule 13.5. */ + xTaskIsRunningOrYielding = taskTASK_IS_RUNNING_OR_SCHEDULED_TO_YIELD( pxTCB ); + + /* If the task is running (or yielding), we must add it to the + * termination list so that an idle task can delete it when it is + * no longer running. */ + if( ( xSchedulerRunning != pdFALSE ) && ( xTaskIsRunningOrYielding != pdFALSE ) ) + { + /* A running task or a task which is scheduled to yield is being + * deleted. This cannot complete when the task is still running + * on a core, as a context switch to another task is required. + * Place the task in the termination list. The idle task will check + * the termination list and free up any memory allocated by the + * scheduler for the TCB and stack of the deleted task. */ + vListInsertEnd( &xTasksWaitingTermination, &( pxTCB->xStateListItem ) ); + + /* Increment the ucTasksDeleted variable so the idle task knows + * there is a task that has been deleted and that it should therefore + * check the xTasksWaitingTermination list. */ + ++uxDeletedTasksWaitingCleanUp; + + /* Call the delete hook before portPRE_TASK_DELETE_HOOK() as + * portPRE_TASK_DELETE_HOOK() does not return in the Win32 port. */ + traceTASK_DELETE( pxTCB ); + + /* Delete the task TCB in idle task. */ + xDeleteTCBInIdleTask = pdTRUE; + + /* The pre-delete hook is primarily for the Windows simulator, + * in which Windows specific clean up operations are performed, + * after which it is not possible to yield away from this task - + * hence xYieldPending is used to latch that a context switch is + * required. */ + #if ( configNUMBER_OF_CORES == 1 ) + portPRE_TASK_DELETE_HOOK( pxTCB, &( xYieldPendings[ 0 ] ) ); + #else + portPRE_TASK_DELETE_HOOK( pxTCB, &( xYieldPendings[ pxTCB->xTaskRunState ] ) ); + #endif + + /* In the case of SMP, it is possible that the task being deleted + * is running on another core. We must evict the task before + * exiting the critical section to ensure that the task cannot + * take an action which puts it back on ready/state/event list, + * thereby nullifying the delete operation. Once evicted, the + * task won't be scheduled ever as it will no longer be on the + * ready list. */ + #if ( configNUMBER_OF_CORES > 1 ) + { + if( taskTASK_IS_RUNNING( pxTCB ) == pdTRUE ) + { + if( pxTCB->xTaskRunState == ( BaseType_t ) portGET_CORE_ID() ) + { + configASSERT( uxSchedulerSuspended == 0 ); + taskYIELD_WITHIN_API(); + } + else + { + prvYieldCore( pxTCB->xTaskRunState ); + } + } + } + #endif /* #if ( configNUMBER_OF_CORES > 1 ) */ + } + else + { + --uxCurrentNumberOfTasks; + traceTASK_DELETE( pxTCB ); + + /* Reset the next expected unblock time in case it referred to + * the task that has just been deleted. */ + prvResetNextTaskUnblockTime(); + } } } - #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ + kernelEXIT_CRITICAL(); + + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + if( xDeferredDeletion == pdFALSE ) + #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + { + /* If the task is not deleting itself, call prvDeleteTCB from outside of + * critical section. If a task deletes itself, prvDeleteTCB is called + * from prvCheckTasksWaitingTermination which is called from Idle task. */ + if( xDeleteTCBInIdleTask != pdTRUE ) + { + prvDeleteTCB( pxTCB ); + } + + /* Force a reschedule if it is the currently running task that has just + * been deleted. */ + #if ( configNUMBER_OF_CORES == 1 ) + { + if( xSchedulerRunning != pdFALSE ) + { + if( pxTCB == pxCurrentTCB ) + { + configASSERT( uxSchedulerSuspended == 0 ); + taskYIELD_WITHIN_API(); + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + } + } + #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ + } traceRETURN_vTaskDelete(); } @@ -2547,14 +2674,14 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, else #endif { - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { pxStateList = listLIST_ITEM_CONTAINER( &( pxTCB->xStateListItem ) ); pxEventList = listLIST_ITEM_CONTAINER( &( pxTCB->xEventListItem ) ); pxDelayedList = pxDelayedTaskList; pxOverflowedDelayedList = pxOverflowDelayedTaskList; } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); if( pxEventList == &xPendingReadyList ) { @@ -2664,7 +2791,15 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, traceENTER_uxTaskPriorityGet( xTask ); - portBASE_TYPE_ENTER_CRITICAL(); + #if ( ( configNUMBER_OF_CORES > 1 ) ) + { + kernelENTER_CRITICAL(); + } + #else + { + portBASE_TYPE_ENTER_CRITICAL(); + } + #endif { /* If null is passed in here then it is the priority of the task * that called uxTaskPriorityGet() that is being queried. */ @@ -2673,7 +2808,15 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, uxReturn = pxTCB->uxPriority; } - portBASE_TYPE_EXIT_CRITICAL(); + #if ( ( configNUMBER_OF_CORES > 1 ) ) + { + kernelEXIT_CRITICAL(); + } + #else + { + portBASE_TYPE_EXIT_CRITICAL(); + } + #endif traceRETURN_uxTaskPriorityGet( uxReturn ); @@ -2714,7 +2857,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, /* MISRA Ref 4.7.1 [Return value shall be checked] */ /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#dir-47 */ /* coverity[misra_c_2012_directive_4_7_violation] */ - uxSavedInterruptStatus = ( UBaseType_t ) taskENTER_CRITICAL_FROM_ISR(); + uxSavedInterruptStatus = ( UBaseType_t ) kernelENTER_CRITICAL_FROM_ISR(); { /* If null is passed in here then it is the priority of the calling * task that is being queried. */ @@ -2723,7 +2866,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, uxReturn = pxTCB->uxPriority; } - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); traceRETURN_uxTaskPriorityGetFromISR( uxReturn ); @@ -2742,7 +2885,15 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, traceENTER_uxTaskBasePriorityGet( xTask ); - portBASE_TYPE_ENTER_CRITICAL(); + #if ( ( configNUMBER_OF_CORES > 1 ) ) + { + kernelENTER_CRITICAL(); + } + #else + { + portBASE_TYPE_ENTER_CRITICAL(); + } + #endif { /* If null is passed in here then it is the base priority of the task * that called uxTaskBasePriorityGet() that is being queried. */ @@ -2751,7 +2902,15 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, uxReturn = pxTCB->uxBasePriority; } - portBASE_TYPE_EXIT_CRITICAL(); + #if ( ( configNUMBER_OF_CORES > 1 ) ) + { + kernelEXIT_CRITICAL(); + } + #else + { + portBASE_TYPE_EXIT_CRITICAL(); + } + #endif traceRETURN_uxTaskBasePriorityGet( uxReturn ); @@ -2792,7 +2951,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, /* MISRA Ref 4.7.1 [Return value shall be checked] */ /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#dir-47 */ /* coverity[misra_c_2012_directive_4_7_violation] */ - uxSavedInterruptStatus = ( UBaseType_t ) taskENTER_CRITICAL_FROM_ISR(); + uxSavedInterruptStatus = ( UBaseType_t ) kernelENTER_CRITICAL_FROM_ISR(); { /* If null is passed in here then it is the base priority of the calling * task that is being queried. */ @@ -2801,7 +2960,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, uxReturn = pxTCB->uxBasePriority; } - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); traceRETURN_uxTaskBasePriorityGetFromISR( uxReturn ); @@ -2838,7 +2997,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, mtCOVERAGE_TEST_MARKER(); } - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { /* If null is passed in here then it is the priority of the calling * task that is being changed. */ @@ -2899,12 +3058,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, /* Setting the priority of a running task down means * there may now be another task of higher priority that * is ready to execute. */ - #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - if( pxTCB->xPreemptionDisable == 0U ) - #endif - { - xYieldRequired = pdTRUE; - } + xYieldRequired = pdTRUE; } else { @@ -3018,7 +3172,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, ( void ) uxPriorityUsedOnEntry; } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); traceRETURN_vTaskPrioritySet(); } @@ -3035,7 +3189,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, traceENTER_vTaskCoreAffinitySet( xTask, uxCoreAffinityMask ); - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { pxTCB = prvGetTCBFromHandle( xTask ); configASSERT( pxTCB != NULL ); @@ -3076,7 +3230,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, } } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); traceRETURN_vTaskCoreAffinitySet(); } @@ -3091,14 +3245,30 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, traceENTER_vTaskCoreAffinityGet( xTask ); - portBASE_TYPE_ENTER_CRITICAL(); + #if ( ( configNUMBER_OF_CORES > 1 ) ) + { + kernelENTER_CRITICAL(); + } + #else + { + portBASE_TYPE_ENTER_CRITICAL(); + } + #endif { pxTCB = prvGetTCBFromHandle( xTask ); configASSERT( pxTCB != NULL ); uxCoreAffinityMask = pxTCB->uxCoreAffinityMask; } - portBASE_TYPE_EXIT_CRITICAL(); + #if ( ( configNUMBER_OF_CORES > 1 ) ) + { + kernelEXIT_CRITICAL(); + } + #else + { + portBASE_TYPE_EXIT_CRITICAL(); + } + #endif traceRETURN_vTaskCoreAffinityGet( uxCoreAffinityMask ); @@ -3116,14 +3286,21 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, traceENTER_vTaskPreemptionDisable( xTask ); - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { - pxTCB = prvGetTCBFromHandle( xTask ); - configASSERT( pxTCB != NULL ); + if( xSchedulerRunning != pdFALSE ) + { + pxTCB = prvGetTCBFromHandle( xTask ); + configASSERT( pxTCB != NULL ); - pxTCB->xPreemptionDisable++; + pxTCB->uxPreemptionDisable++; + } + else + { + mtCOVERAGE_TEST_MARKER(); + } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); traceRETURN_vTaskPreemptionDisable(); } @@ -3133,31 +3310,86 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - void vTaskPreemptionEnable( const TaskHandle_t xTask ) + BaseType_t prvTaskPreemptionEnable( const TaskHandle_t xTask ) { TCB_t * pxTCB; - BaseType_t xCoreID; + UBaseType_t uxDeferredAction = 0U; + BaseType_t xAlreadyYielded = pdFALSE; traceENTER_vTaskPreemptionEnable( xTask ); - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { - pxTCB = prvGetTCBFromHandle( xTask ); - configASSERT( pxTCB != NULL ); - configASSERT( pxTCB->xPreemptionDisable > 0U ); - - pxTCB->xPreemptionDisable--; - if( xSchedulerRunning != pdFALSE ) { - if( ( pxTCB->xPreemptionDisable == 0U ) && ( taskTASK_IS_RUNNING( pxTCB ) == pdTRUE ) ) + pxTCB = prvGetTCBFromHandle( xTask ); + configASSERT( pxTCB != NULL ); + configASSERT( pxTCB->uxPreemptionDisable > 0U ); + + pxTCB->uxPreemptionDisable--; + + if( pxTCB->uxPreemptionDisable == 0U ) { - xCoreID = ( BaseType_t ) pxTCB->xTaskRunState; - prvYieldCore( xCoreID ); + if( pxTCB->uxDeferredStateChange != 0U ) + { + uxDeferredAction = pxTCB->uxDeferredStateChange; + } + else + { + if( ( xYieldPendings[ pxTCB->xTaskRunState ] != pdFALSE ) && ( taskTASK_IS_RUNNING( pxTCB ) != pdFALSE ) ) + { + prvYieldCore( pxTCB->xTaskRunState ); + xAlreadyYielded = pdTRUE; + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + } + } + else + { + mtCOVERAGE_TEST_MARKER(); } } + else + { + mtCOVERAGE_TEST_MARKER(); + } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); + + if( uxDeferredAction != 0U ) + { + if( uxDeferredAction & tskDEFERRED_DELETION ) + { + vTaskDelete( xTask ); + } + else if( uxDeferredAction & tskDEFERRED_SUSPENSION ) + { + vTaskSuspend( xTask ); + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + + /* Any deferred action on the task would result in a context switch. */ + xAlreadyYielded = pdTRUE; + } + + return xAlreadyYielded; + } +#endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ +/*-----------------------------------------------------------*/ + +#if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + + void vTaskPreemptionEnable( const TaskHandle_t xTask ) + { + traceENTER_vTaskPreemptionEnable( xTask ); + + ( void ) prvTaskPreemptionEnable( xTask ); traceRETURN_vTaskPreemptionEnable(); } @@ -3171,82 +3403,110 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, { TCB_t * pxTCB; + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + BaseType_t xDeferredSuspension = pdFALSE; + #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + traceENTER_vTaskSuspend( xTaskToSuspend ); - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { /* If null is passed in here then it is the running task that is * being suspended. */ pxTCB = prvGetTCBFromHandle( xTaskToSuspend ); configASSERT( pxTCB != NULL ); - traceTASK_SUSPEND( pxTCB ); + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - /* Remove task from the ready/delayed list and place in the - * suspended list. */ - if( uxListRemove( &( pxTCB->xStateListItem ) ) == ( UBaseType_t ) 0 ) - { - taskRESET_READY_PRIORITY( pxTCB->uxPriority ); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } - - /* Is the task waiting on an event also? */ - if( listLIST_ITEM_CONTAINER( &( pxTCB->xEventListItem ) ) != NULL ) - { - ( void ) uxListRemove( &( pxTCB->xEventListItem ) ); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } - - vListInsertEnd( &xSuspendedTaskList, &( pxTCB->xStateListItem ) ); - - #if ( configUSE_TASK_NOTIFICATIONS == 1 ) - { - BaseType_t x; - - for( x = ( BaseType_t ) 0; x < ( BaseType_t ) configTASK_NOTIFICATION_ARRAY_ENTRIES; x++ ) + /* If the task has disabled preemption, we need to defer the suspension until the + * task enables preemption. The suspension will be performed in vTaskPreemptionEnable(). */ + if( pxTCB->uxPreemptionDisable > 0U ) { - if( pxTCB->ucNotifyState[ x ] == taskWAITING_NOTIFICATION ) + pxTCB->uxDeferredStateChange |= tskDEFERRED_SUSPENSION; + xDeferredSuspension = pdTRUE; + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + #endif /* configUSE_TASK_PREEMPTION_DISABLE */ + + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + if( xDeferredSuspension == pdFALSE ) + #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + { + traceTASK_SUSPEND( pxTCB ); + + /* Remove task from the ready/delayed list and place in the + * suspended list. */ + if( uxListRemove( &( pxTCB->xStateListItem ) ) == ( UBaseType_t ) 0 ) + { + taskRESET_READY_PRIORITY( pxTCB->uxPriority ); + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + + /* Is the task waiting on an event also? */ + if( listLIST_ITEM_CONTAINER( &( pxTCB->xEventListItem ) ) != NULL ) + { + ( void ) uxListRemove( &( pxTCB->xEventListItem ) ); + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + + vListInsertEnd( &xSuspendedTaskList, &( pxTCB->xStateListItem ) ); + + #if ( configUSE_TASK_NOTIFICATIONS == 1 ) + { + BaseType_t x; + + for( x = ( BaseType_t ) 0; x < ( BaseType_t ) configTASK_NOTIFICATION_ARRAY_ENTRIES; x++ ) { - /* The task was blocked to wait for a notification, but is - * now suspended, so no notification was received. */ - pxTCB->ucNotifyState[ x ] = taskNOT_WAITING_NOTIFICATION; + if( pxTCB->ucNotifyState[ x ] == taskWAITING_NOTIFICATION ) + { + /* The task was blocked to wait for a notification, but is + * now suspended, so no notification was received. */ + pxTCB->ucNotifyState[ x ] = taskNOT_WAITING_NOTIFICATION; + } } } - } - #endif /* if ( configUSE_TASK_NOTIFICATIONS == 1 ) */ + #endif /* if ( configUSE_TASK_NOTIFICATIONS == 1 ) */ - /* In the case of SMP, it is possible that the task being suspended - * is running on another core. We must evict the task before - * exiting the critical section to ensure that the task cannot - * take an action which puts it back on ready/state/event list, - * thereby nullifying the suspend operation. Once evicted, the - * task won't be scheduled before it is resumed as it will no longer - * be on the ready list. */ - #if ( configNUMBER_OF_CORES > 1 ) - { - if( xSchedulerRunning != pdFALSE ) + /* In the case of SMP, it is possible that the task being suspended + * is running on another core. We must evict the task before + * exiting the critical section to ensure that the task cannot + * take an action which puts it back on ready/state/event list, + * thereby nullifying the suspend operation. Once evicted, the + * task won't be scheduled before it is resumed as it will no longer + * be on the ready list. */ + #if ( configNUMBER_OF_CORES > 1 ) { - /* Reset the next expected unblock time in case it referred to the - * task that is now in the Suspended state. */ - prvResetNextTaskUnblockTime(); - - if( taskTASK_IS_RUNNING( pxTCB ) == pdTRUE ) + if( xSchedulerRunning != pdFALSE ) { - if( pxTCB->xTaskRunState == ( BaseType_t ) portGET_CORE_ID() ) + /* Reset the next expected unblock time in case it referred to the + * task that is now in the Suspended state. */ + prvResetNextTaskUnblockTime(); + + if( taskTASK_IS_RUNNING( pxTCB ) == pdTRUE ) { - /* The current task has just been suspended. */ - configASSERT( uxSchedulerSuspended == 0 ); - vTaskYieldWithinAPI(); + if( pxTCB->xTaskRunState == ( BaseType_t ) portGET_CORE_ID() ) + { + /* The current task has just been suspended. */ + configASSERT( uxSchedulerSuspended == 0 ); + vTaskYieldWithinAPI(); + } + else + { + prvYieldCore( pxTCB->xTaskRunState ); + } } else { - prvYieldCore( pxTCB->xTaskRunState ); + mtCOVERAGE_TEST_MARKER(); } } else @@ -3254,73 +3514,74 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, mtCOVERAGE_TEST_MARKER(); } } + #endif /* #if ( configNUMBER_OF_CORES > 1 ) */ + } + } + kernelEXIT_CRITICAL(); + + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + if( xDeferredSuspension == pdFALSE ) + #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + { + #if ( configNUMBER_OF_CORES == 1 ) + { + UBaseType_t uxCurrentListLength; + + if( xSchedulerRunning != pdFALSE ) + { + /* Reset the next expected unblock time in case it referred to the + * task that is now in the Suspended state. */ + kernelENTER_CRITICAL(); + { + prvResetNextTaskUnblockTime(); + } + kernelEXIT_CRITICAL(); + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + + if( pxTCB == pxCurrentTCB ) + { + if( xSchedulerRunning != pdFALSE ) + { + /* The current task has just been suspended. */ + configASSERT( uxSchedulerSuspended == 0 ); + portYIELD_WITHIN_API(); + } + else + { + /* The scheduler is not running, but the task that was pointed + * to by pxCurrentTCB has just been suspended and pxCurrentTCB + * must be adjusted to point to a different task. */ + + /* Use a temp variable as a distinct sequence point for reading + * volatile variables prior to a comparison to ensure compliance + * with MISRA C 2012 Rule 13.2. */ + uxCurrentListLength = listCURRENT_LIST_LENGTH( &xSuspendedTaskList ); + + if( uxCurrentListLength == uxCurrentNumberOfTasks ) + { + /* No other tasks are ready, so set pxCurrentTCB back to + * NULL so when the next task is created pxCurrentTCB will + * be set to point to it no matter what its relative priority + * is. */ + pxCurrentTCB = NULL; + } + else + { + vTaskSwitchContext(); + } + } + } else { mtCOVERAGE_TEST_MARKER(); } } - #endif /* #if ( configNUMBER_OF_CORES > 1 ) */ + #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ } - taskEXIT_CRITICAL(); - - #if ( configNUMBER_OF_CORES == 1 ) - { - UBaseType_t uxCurrentListLength; - - if( xSchedulerRunning != pdFALSE ) - { - /* Reset the next expected unblock time in case it referred to the - * task that is now in the Suspended state. */ - taskENTER_CRITICAL(); - { - prvResetNextTaskUnblockTime(); - } - taskEXIT_CRITICAL(); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } - - if( pxTCB == pxCurrentTCB ) - { - if( xSchedulerRunning != pdFALSE ) - { - /* The current task has just been suspended. */ - configASSERT( uxSchedulerSuspended == 0 ); - portYIELD_WITHIN_API(); - } - else - { - /* The scheduler is not running, but the task that was pointed - * to by pxCurrentTCB has just been suspended and pxCurrentTCB - * must be adjusted to point to a different task. */ - - /* Use a temp variable as a distinct sequence point for reading - * volatile variables prior to a comparison to ensure compliance - * with MISRA C 2012 Rule 13.2. */ - uxCurrentListLength = listCURRENT_LIST_LENGTH( &xSuspendedTaskList ); - - if( uxCurrentListLength == uxCurrentNumberOfTasks ) - { - /* No other tasks are ready, so set pxCurrentTCB back to - * NULL so when the next task is created pxCurrentTCB will - * be set to point to it no matter what its relative priority - * is. */ - pxCurrentTCB = NULL; - } - else - { - vTaskSwitchContext(); - } - } - } - else - { - mtCOVERAGE_TEST_MARKER(); - } - } - #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ traceRETURN_vTaskSuspend(); } @@ -3404,6 +3665,10 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, { TCB_t * const pxTCB = xTaskToResume; + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + BaseType_t xTaskResumed = pdFALSE; + #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + traceENTER_vTaskResume( xTaskToResume ); /* It does not make sense to resume the calling task. */ @@ -3424,28 +3689,48 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, if( pxTCB != NULL ) #endif { - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { - if( prvTaskIsTaskSuspended( pxTCB ) != pdFALSE ) - { - traceTASK_RESUME( pxTCB ); + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - /* The ready list can be accessed even if the scheduler is - * suspended because this is inside a critical section. */ - ( void ) uxListRemove( &( pxTCB->xStateListItem ) ); - prvAddTaskToReadyList( pxTCB ); + /* If the task being resumed is in a deferred suspension state, + * we simply clear the deferred suspension state and return. */ + if( pxTCB->uxDeferredStateChange & tskDEFERRED_SUSPENSION ) + { + pxTCB->uxDeferredStateChange &= ~tskDEFERRED_SUSPENSION; + xTaskResumed = pdTRUE; + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + #endif /* configUSE_TASK_PREEMPTION_DISABLE */ - /* This yield may not cause the task just resumed to run, - * but will leave the lists in the correct state for the - * next yield. */ - taskYIELD_ANY_CORE_IF_USING_PREEMPTION( pxTCB ); - } - else + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + if( xTaskResumed == pdFALSE ) + #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ { - mtCOVERAGE_TEST_MARKER(); + if( prvTaskIsTaskSuspended( pxTCB ) != pdFALSE ) + { + traceTASK_RESUME( pxTCB ); + + /* The ready list can be accessed even if the scheduler is + * suspended because this is inside a critical section. */ + ( void ) uxListRemove( &( pxTCB->xStateListItem ) ); + prvAddTaskToReadyList( pxTCB ); + + /* This yield may not cause the task just resumed to run, + * but will leave the lists in the correct state for the + * next yield. */ + taskYIELD_ANY_CORE_IF_USING_PREEMPTION( pxTCB ); + } + else + { + mtCOVERAGE_TEST_MARKER(); + } } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); } else { @@ -3492,7 +3777,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, /* MISRA Ref 4.7.1 [Return value shall be checked] */ /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#dir-47 */ /* coverity[misra_c_2012_directive_4_7_violation] */ - uxSavedInterruptStatus = taskENTER_CRITICAL_FROM_ISR(); + uxSavedInterruptStatus = kernelENTER_CRITICAL_FROM_ISR(); { if( prvTaskIsTaskSuspended( pxTCB ) != pdFALSE ) { @@ -3548,7 +3833,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, mtCOVERAGE_TEST_MARKER(); } } - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); traceRETURN_xTaskResumeFromISR( xYieldRequired ); @@ -3930,45 +4215,39 @@ void vTaskSuspendAll( void ) * do not otherwise exhibit real time behaviour. */ portSOFTWARE_BARRIER(); - #if ( portUSING_GRANULAR_LOCKS == 1 ) + kernelGET_TASK_LOCK( xCoreID ); + + /* uxSchedulerSuspended is increased after prvCheckForRunStateChange. The + * purpose is to prevent altering the variable when fromISR APIs are readying + * it. */ + if( ( uxSchedulerSuspended == 0U ) + + /* If the scheduler is being suspended after taking a granular lock (in any data group) + * then we do not check for the run state of a task and we let it run through until + * the granular lock is released. */ + #if ( portUSING_GRANULAR_LOCKS == 1 ) + && ( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + ) { - portGET_SPINLOCK( &xTaskSpinlock ); - portGET_SPINLOCK( &xISRSpinlock ); - - /* Increment xPreemptionDisable to prevent preemption and also - * track whether to yield in xTaskResumeAll(). */ - pxCurrentTCBs[ portGET_CORE_ID() ]->xPreemptionDisable++; - - /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment - * is used to allow calls to vTaskSuspendAll() to nest. */ - ++uxSchedulerSuspended; - - portRELEASE_SPINLOCK( &xISRSpinlock ); + prvCheckForRunStateChange(); } - #else /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + else { - portGET_TASK_LOCK(); - - /* uxSchedulerSuspended is increased after prvCheckForRunStateChange. The - * purpose is to prevent altering the variable when fromISR APIs are readying - * it. */ - if( uxSchedulerSuspended == 0U ) - { - prvCheckForRunStateChange(); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } - - portGET_ISR_LOCK(); - - /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment - * is used to allow calls to vTaskSuspendAll() to nest. */ - ++uxSchedulerSuspended; - portRELEASE_ISR_LOCK(); + mtCOVERAGE_TEST_MARKER(); } - #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + + /* Query the coreID again as prvCheckForRunStateChange may have + * caused the task to get scheduled on a different core. The correct + * task lock for the core is acquired in prvCheckForRunStateChange. */ + xCoreID = ( BaseType_t ) portGET_CORE_ID(); + + kernelGET_ISR_LOCK( xCoreID ); + + /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment + * is used to allow calls to vTaskSuspendAll() to nest. */ + ++uxSchedulerSuspended; + kernelRELEASE_ISR_LOCK( xCoreID ); portCLEAR_INTERRUPT_MASK( ulState ); } @@ -4064,7 +4343,7 @@ BaseType_t xTaskResumeAll( void ) * removed task will have been added to the xPendingReadyList. Once the * scheduler has been resumed it is safe to move all the pending ready * tasks from this list into their appropriate ready list. */ - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); @@ -4073,24 +4352,9 @@ BaseType_t xTaskResumeAll( void ) configASSERT( uxSchedulerSuspended != 0U ); uxSchedulerSuspended = ( UBaseType_t ) ( uxSchedulerSuspended - 1U ); - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - { - configASSERT( pxCurrentTCBs[ portGET_CORE_ID() ]->xPreemptionDisable > 0 ); - - /* Decrement xPreemptionDisable. If 0, it means this we are not - * in a nested suspension scenario, thus this function and yield - * if necessary. */ - pxCurrentTCBs[ portGET_CORE_ID() ]->xPreemptionDisable--; - - /* Release the kernel's task spinlock that was held throughout - * the kernel suspension. */ - portRELEASE_SPINLOCK( &xTaskSpinlock ); - } - #else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ - { - portRELEASE_TASK_LOCK(); - } - #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #if ( configNUMBER_OF_CORES > 1 ) + kernelRELEASE_TASK_LOCK( xCoreID ); + #endif if( uxSchedulerSuspended == ( UBaseType_t ) 0U ) { @@ -4205,7 +4469,7 @@ BaseType_t xTaskResumeAll( void ) mtCOVERAGE_TEST_MARKER(); } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); } traceRETURN_xTaskResumeAll( xAlreadyYielded ); @@ -4634,11 +4898,11 @@ char * pcTaskGetName( TaskHandle_t xTaskToQuery ) configASSERT( xTicksToJump != ( TickType_t ) 0 ); /* Prevent the tick interrupt modifying xPendedTicks simultaneously. */ - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { xPendedTicks++; } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); xTicksToJump--; } else @@ -4670,11 +4934,11 @@ BaseType_t xTaskCatchUpTicks( TickType_t xTicksToCatchUp ) vTaskSuspendAll(); /* Prevent the tick interrupt modifying xPendedTicks simultaneously. */ - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { xPendedTicks += xTicksToCatchUp; } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); xYieldOccurred = xTaskResumeAll(); traceRETURN_xTaskCatchUpTicks( xYieldOccurred ); @@ -4711,7 +4975,7 @@ BaseType_t xTaskCatchUpTicks( TickType_t xTicksToCatchUp ) * the event list too. Interrupts can touch the event list item, * even though the scheduler is suspended, so a critical section * is used. */ - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { if( listLIST_ITEM_CONTAINER( &( pxTCB->xEventListItem ) ) != NULL ) { @@ -4727,7 +4991,7 @@ BaseType_t xTaskCatchUpTicks( TickType_t xTicksToCatchUp ) mtCOVERAGE_TEST_MARKER(); } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); /* Place the unblocked task into the appropriate ready list. */ prvAddTaskToReadyList( pxTCB ); @@ -4754,11 +5018,11 @@ BaseType_t xTaskCatchUpTicks( TickType_t xTicksToCatchUp ) } #else /* #if ( configNUMBER_OF_CORES == 1 ) */ { - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { prvYieldForTask( pxTCB ); } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); } #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ } @@ -4785,8 +5049,16 @@ BaseType_t xTaskIncrementTick( void ) TickType_t xItemValue; BaseType_t xSwitchRequired = pdFALSE; + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + UBaseType_t uxSavedInterruptStatus; + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceENTER_xTaskIncrementTick(); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + uxSavedInterruptStatus = kernelENTER_CRITICAL_FROM_ISR(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + /* Called by the portable layer each time a tick interrupt occurs. * Increments the tick then checks to see if the new tick value will cause any * tasks to be unblocked. */ @@ -4887,7 +5159,8 @@ BaseType_t xTaskIncrementTick( void ) /* Preemption is on, but a context switch should * only be performed if the unblocked task's * priority is higher than the currently executing - * task. + * task and the currently executing task does not + * have preemption disabled. * The case of equal priority tasks sharing * processing time (which happens when both * preemption and time slicing are on) is @@ -4985,7 +5258,7 @@ BaseType_t xTaskIncrementTick( void ) for( xCoreID = 0; xCoreID < ( BaseType_t ) configNUMBER_OF_CORES; xCoreID++ ) { #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - if( pxCurrentTCBs[ xCoreID ]->xPreemptionDisable == 0U ) + if( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) #endif { if( xYieldPendings[ xCoreID ] != pdFALSE ) @@ -5023,6 +5296,10 @@ BaseType_t xTaskIncrementTick( void ) #endif } + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceRETURN_xTaskIncrementTick( xSwitchRequired ); return xSwitchRequired; @@ -5051,11 +5328,11 @@ BaseType_t xTaskIncrementTick( void ) /* Save the hook function in the TCB. A critical section is required as * the value can be accessed from an interrupt. */ - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { xTCB->pxTaskTag = pxHookFunction; } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); traceRETURN_vTaskSetApplicationTaskTag(); } @@ -5078,11 +5355,11 @@ BaseType_t xTaskIncrementTick( void ) /* Save the hook function in the TCB. A critical section is required as * the value can be accessed from an interrupt. */ - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { xReturn = pxTCB->pxTaskTag; } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); traceRETURN_xTaskGetApplicationTaskTag( xReturn ); @@ -5111,11 +5388,11 @@ BaseType_t xTaskIncrementTick( void ) /* MISRA Ref 4.7.1 [Return value shall be checked] */ /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#dir-47 */ /* coverity[misra_c_2012_directive_4_7_violation] */ - uxSavedInterruptStatus = taskENTER_CRITICAL_FROM_ISR(); + uxSavedInterruptStatus = kernelENTER_CRITICAL_FROM_ISR(); { xReturn = pxTCB->pxTaskTag; } - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); traceRETURN_xTaskGetApplicationTaskTagFromISR( xReturn ); @@ -5260,18 +5537,25 @@ BaseType_t xTaskIncrementTick( void ) * and move on if another core suspended the scheduler. We should only * do that if the current core has suspended the scheduler. */ - portGET_TASK_LOCK( xCoreID ); /* Must always acquire the task lock first. */ - portGET_ISR_LOCK( xCoreID ); + kernelGET_TASK_LOCK( xCoreID ); /* Must always acquire the task lock first. */ + kernelGET_ISR_LOCK( xCoreID ); { /* vTaskSwitchContext() must never be called from within a critical section. * This is not necessarily true for single core FreeRTOS, but it is for this * SMP port. */ configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 ); + /* vTaskSwitchContext() must not be called with a task that has + * preemption disabled. */ + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + configASSERT( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ); + #endif + if( uxSchedulerSuspended != ( UBaseType_t ) 0U ) { - /* The scheduler is currently suspended - do not allow a context - * switch. */ + /* The scheduler is currently suspended or the task + * has requested to not be preempted - do not allow + * a context switch. */ xYieldPendings[ xCoreID ] = pdTRUE; } else @@ -5342,8 +5626,8 @@ BaseType_t xTaskIncrementTick( void ) #endif } } - portRELEASE_ISR_LOCK( xCoreID ); - portRELEASE_TASK_LOCK( xCoreID ); + kernelRELEASE_ISR_LOCK( xCoreID ); + kernelRELEASE_TASK_LOCK( xCoreID ); traceRETURN_vTaskSwitchContext(); } @@ -5357,8 +5641,15 @@ void vTaskPlaceOnEventList( List_t * const pxEventList, configASSERT( pxEventList ); - /* THIS FUNCTION MUST BE CALLED WITH THE - * SCHEDULER SUSPENDED AND THE QUEUE BEING ACCESSED LOCKED. */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* Suspend the kernel data group as we are about to access its members */ + vTaskSuspendAll(); + #else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + + /* THIS FUNCTION MUST BE CALLED WITH THE + * SCHEDULER SUSPENDED AND THE QUEUE BEING ACCESSED LOCKED. */ + configASSERT( uxSchedulerSuspended != ( UBaseType_t ) 0U ); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /* Place the event list item of the TCB in the appropriate event list. * This is placed in the list in priority order so the highest priority task @@ -5375,6 +5666,11 @@ void vTaskPlaceOnEventList( List_t * const pxEventList, prvAddCurrentTaskToDelayedList( xTicksToWait, pdTRUE ); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* We are done accessing the kernel data group. Resume it. */ + ( void ) xTaskResumeAll(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceRETURN_vTaskPlaceOnEventList(); } /*-----------------------------------------------------------*/ @@ -5387,9 +5683,16 @@ void vTaskPlaceOnUnorderedEventList( List_t * pxEventList, configASSERT( pxEventList ); - /* THIS FUNCTION MUST BE CALLED WITH THE SCHEDULER SUSPENDED. It is used by - * the event groups implementation. */ - configASSERT( uxSchedulerSuspended != ( UBaseType_t ) 0U ); + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* Suspend the kernel data group as we are about to access its members */ + vTaskSuspendAll(); + #else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + + /* THIS FUNCTION MUST BE CALLED WITH THE SCHEDULER SUSPENDED. It is used by + * the event groups implementation. */ + configASSERT( uxSchedulerSuspended != ( UBaseType_t ) 0U ); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /* Store the item value in the event list item. It is safe to access the * event list item here as interrupts won't access the event list item of a @@ -5405,6 +5708,11 @@ void vTaskPlaceOnUnorderedEventList( List_t * pxEventList, prvAddCurrentTaskToDelayedList( xTicksToWait, pdTRUE ); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* We are done accessing the kernel data group. Resume it. */ + ( void ) xTaskResumeAll(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceRETURN_vTaskPlaceOnUnorderedEventList(); } /*-----------------------------------------------------------*/ @@ -5419,11 +5727,17 @@ void vTaskPlaceOnUnorderedEventList( List_t * pxEventList, configASSERT( pxEventList ); - /* This function should not be called by application code hence the - * 'Restricted' in its name. It is not part of the public API. It is - * designed for use by kernel code, and has special calling requirements - - * it should be called with the scheduler suspended. */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* Suspend the kernel data group as we are about to access its members */ + vTaskSuspendAll(); + #else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + /* This function should not be called by application code hence the + * 'Restricted' in its name. It is not part of the public API. It is + * designed for use by kernel code, and has special calling requirements - + * it should be called with the scheduler suspended. */ + configASSERT( uxSchedulerSuspended != ( UBaseType_t ) 0U ); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /* Place the event list item of the TCB in the appropriate event list. * In this case it is assume that this is the only task that is going to @@ -5442,6 +5756,11 @@ void vTaskPlaceOnUnorderedEventList( List_t * pxEventList, traceTASK_DELAY_UNTIL( ( xTickCount + xTicksToWait ) ); prvAddCurrentTaskToDelayedList( xTicksToWait, xWaitIndefinitely ); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* We are done accessing the kernel data group. Resume it. */ + ( void ) xTaskResumeAll(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceRETURN_vTaskPlaceOnEventListRestricted(); } @@ -5453,10 +5772,34 @@ BaseType_t xTaskRemoveFromEventList( const List_t * const pxEventList ) TCB_t * pxUnblockedTCB; BaseType_t xReturn; + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + UBaseType_t uxSavedInterruptStatus; + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceENTER_xTaskRemoveFromEventList( pxEventList ); - /* THIS FUNCTION MUST BE CALLED FROM A CRITICAL SECTION. It can also be - * called from a critical section within an ISR. */ + #if ( !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) ) + + /* THIS FUNCTION MUST BE CALLED FROM A CRITICAL SECTION. It can also be + * called from a critical section within an ISR. */ + #else /* #if ( ! ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) ) */ + /* Lock the kernel data group as we are about to access its members */ + if( portCHECK_IF_IN_ISR() == pdTRUE ) + { + uxSavedInterruptStatus = kernelENTER_CRITICAL_FROM_ISR(); + } + else + { + uxSavedInterruptStatus = 0; + kernelENTER_CRITICAL(); + } + + /* Before taking the kernel lock, another task/ISR could have already + * emptied the pxEventList. So we insert a check here to see if + * pxEventList is empty before attempting to remove an item from it. */ + if( listLIST_IS_EMPTY( pxEventList ) == pdFALSE ) + { + #endif /* #if ( ! ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) ) */ /* The event list is sorted in priority order, so the first in the list can * be removed as it is known to be the highest priority. Remove the TCB from @@ -5536,6 +5879,26 @@ BaseType_t xTaskRemoveFromEventList( const List_t * const pxEventList ) } #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) +} +else +{ + /* The pxEventList was emptied before we entered the critical + * section, Nothing to do except return pdFALSE. */ + xReturn = pdFALSE; +} + +/* We are done accessing the kernel data group. Unlock it. */ +if( portCHECK_IF_IN_ISR() == pdTRUE ) +{ + kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); +} +else +{ + kernelEXIT_CRITICAL(); +} + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceRETURN_xTaskRemoveFromEventList( xReturn ); return xReturn; } @@ -5599,13 +5962,13 @@ void vTaskRemoveFromUnorderedEventList( ListItem_t * pxEventListItem, { #if ( configUSE_PREEMPTION == 1 ) { - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { prvYieldForTask( pxUnblockedTCB ); } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); } - #endif + #endif /* if ( configUSE_PREEMPTION == 1 ) */ } #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ @@ -5618,12 +5981,12 @@ void vTaskSetTimeOutState( TimeOut_t * const pxTimeOut ) traceENTER_vTaskSetTimeOutState( pxTimeOut ); configASSERT( pxTimeOut ); - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { pxTimeOut->xOverflowCount = xNumOfOverflows; pxTimeOut->xTimeOnEntering = xTickCount; } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); traceRETURN_vTaskSetTimeOutState(); } @@ -5633,10 +5996,20 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) { traceENTER_vTaskInternalSetTimeOutState( pxTimeOut ); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* Lock the kernel data group as we are about to access its members */ + kernelENTER_CRITICAL(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + /* For internal use only as it does not use a critical section. */ pxTimeOut->xOverflowCount = xNumOfOverflows; pxTimeOut->xTimeOnEntering = xTickCount; + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* We are done accessing the kernel data group. Unlock it. */ + kernelEXIT_CRITICAL(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceRETURN_vTaskInternalSetTimeOutState(); } /*-----------------------------------------------------------*/ @@ -5651,7 +6024,7 @@ BaseType_t xTaskCheckForTimeOut( TimeOut_t * const pxTimeOut, configASSERT( pxTimeOut ); configASSERT( pxTicksToWait ); - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { /* Minor optimisation. The tick count cannot change in this block. */ const TickType_t xConstTickCount = xTickCount; @@ -5702,7 +6075,7 @@ BaseType_t xTaskCheckForTimeOut( TimeOut_t * const pxTimeOut, xReturn = pdTRUE; } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); traceRETURN_xTaskCheckForTimeOut( xReturn ); @@ -6004,7 +6377,12 @@ STATIC portTASK_FUNCTION( prvIdleTask, traceENTER_eTaskConfirmSleepModeStatus(); - /* This function must be called from a critical section. */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* Lock the kernel data group as we are about to access its members */ + kernelENTER_CRITICAL(); + #else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + /* This function must be called from a critical section. */ + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ if( listCURRENT_LIST_LENGTH( &xPendingReadyList ) != 0U ) { @@ -6038,6 +6416,11 @@ STATIC portTASK_FUNCTION( prvIdleTask, mtCOVERAGE_TEST_MARKER(); } + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* We are done accessing the kernel data group. Unlock it. */ + kernelEXIT_CRITICAL(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceRETURN_eTaskConfirmSleepModeStatus( eReturn ); return eReturn; @@ -6169,7 +6552,7 @@ STATIC void prvCheckTasksWaitingTermination( void ) { #if ( configNUMBER_OF_CORES == 1 ) { - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { { /* MISRA Ref 11.5.3 [Void pointer assignment] */ @@ -6181,7 +6564,7 @@ STATIC void prvCheckTasksWaitingTermination( void ) --uxDeletedTasksWaitingCleanUp; } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); prvDeleteTCB( pxTCB ); } @@ -6189,7 +6572,7 @@ STATIC void prvCheckTasksWaitingTermination( void ) { pxTCB = NULL; - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { /* For SMP, multiple idles can be running simultaneously * and we need to check that other idles did not cleanup while we were @@ -6212,12 +6595,12 @@ STATIC void prvCheckTasksWaitingTermination( void ) /* The TCB to be deleted still has not yet been switched out * by the scheduler, so we will just exit this loop early and * try again next time. */ - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); break; } } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); if( pxTCB != NULL ) { @@ -6339,14 +6722,14 @@ STATIC void prvCheckTasksWaitingTermination( void ) /* Tasks can be in pending ready list and other state list at the * same time. These tasks are in ready state no matter what state * list the task is in. */ - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { if( listIS_CONTAINED_WITHIN( &xPendingReadyList, &( pxTCB->xEventListItem ) ) != pdFALSE ) { pxTaskStatus->eCurrentState = eReady; } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); } } else @@ -6668,7 +7051,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) else { #if ( configNUMBER_OF_CORES > 1 ) - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); #endif { if( uxSchedulerSuspended == ( UBaseType_t ) 0U ) @@ -6681,7 +7064,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) } } #if ( configNUMBER_OF_CORES > 1 ) - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); #endif } @@ -6702,7 +7085,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) traceENTER_xTaskPriorityInherit( pxMutexHolder ); - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { /* If the mutex is taken by an interrupt, the mutex holder is NULL. Priority * inheritance is not applied in this scenario. */ @@ -6790,7 +7173,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) mtCOVERAGE_TEST_MARKER(); } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); traceRETURN_xTaskPriorityInherit( xReturn ); @@ -6809,6 +7192,11 @@ STATIC void prvResetNextTaskUnblockTime( void ) traceENTER_xTaskPriorityDisinherit( pxMutexHolder ); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* Lock the kernel data group as we are about to access its members */ + kernelENTER_CRITICAL(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + if( pxMutexHolder != NULL ) { /* A task can only have an inherited priority if it holds the mutex. @@ -6886,6 +7274,11 @@ STATIC void prvResetNextTaskUnblockTime( void ) mtCOVERAGE_TEST_MARKER(); } + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* We are done accessing the kernel data group. Unlock it. */ + kernelEXIT_CRITICAL(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceRETURN_xTaskPriorityDisinherit( xReturn ); return xReturn; @@ -6905,88 +7298,97 @@ STATIC void prvResetNextTaskUnblockTime( void ) traceENTER_vTaskPriorityDisinheritAfterTimeout( pxMutexHolder, uxHighestPriorityWaitingTask ); - if( pxMutexHolder != NULL ) + #if ( portUSING_GRANULAR_LOCKS == 1 ) + kernelENTER_CRITICAL(); + #endif { - /* If pxMutexHolder is not NULL then the holder must hold at least - * one mutex. */ - configASSERT( pxTCB->uxMutexesHeld ); + if( pxMutexHolder != NULL ) + { + /* If pxMutexHolder is not NULL then the holder must hold at least + * one mutex. */ + configASSERT( pxTCB->uxMutexesHeld ); - /* Determine the priority to which the priority of the task that - * holds the mutex should be set. This will be the greater of the - * holding task's base priority and the priority of the highest - * priority task that is waiting to obtain the mutex. */ - if( pxTCB->uxBasePriority < uxHighestPriorityWaitingTask ) - { - uxPriorityToUse = uxHighestPriorityWaitingTask; - } - else - { - uxPriorityToUse = pxTCB->uxBasePriority; - } - - /* Does the priority need to change? */ - if( pxTCB->uxPriority != uxPriorityToUse ) - { - /* Only disinherit if no other mutexes are held. This is a - * simplification in the priority inheritance implementation. If - * the task that holds the mutex is also holding other mutexes then - * the other mutexes may have caused the priority inheritance. */ - if( pxTCB->uxMutexesHeld == uxOnlyOneMutexHeld ) + /* Determine the priority to which the priority of the task that + * holds the mutex should be set. This will be the greater of the + * holding task's base priority and the priority of the highest + * priority task that is waiting to obtain the mutex. */ + if( pxTCB->uxBasePriority < uxHighestPriorityWaitingTask ) { - /* If a task has timed out because it already holds the - * mutex it was trying to obtain then it cannot of inherited - * its own priority. */ - configASSERT( pxTCB != pxCurrentTCB ); + uxPriorityToUse = uxHighestPriorityWaitingTask; + } + else + { + uxPriorityToUse = pxTCB->uxBasePriority; + } - /* Disinherit the priority, remembering the previous - * priority to facilitate determining the subject task's - * state. */ - traceTASK_PRIORITY_DISINHERIT( pxTCB, uxPriorityToUse ); - uxPriorityUsedOnEntry = pxTCB->uxPriority; - pxTCB->uxPriority = uxPriorityToUse; + /* Does the priority need to change? */ + if( pxTCB->uxPriority != uxPriorityToUse ) + { + /* Only disinherit if no other mutexes are held. This is a + * simplification in the priority inheritance implementation. If + * the task that holds the mutex is also holding other mutexes then + * the other mutexes may have caused the priority inheritance. */ + if( pxTCB->uxMutexesHeld == uxOnlyOneMutexHeld ) + { + /* If a task has timed out because it already holds the + * mutex it was trying to obtain then it cannot of inherited + * its own priority. */ + configASSERT( pxTCB != pxCurrentTCB ); - /* Only reset the event list item value if the value is not - * being used for anything else. */ - if( ( listGET_LIST_ITEM_VALUE( &( pxTCB->xEventListItem ) ) & taskEVENT_LIST_ITEM_VALUE_IN_USE ) == ( ( TickType_t ) 0U ) ) - { - listSET_LIST_ITEM_VALUE( &( pxTCB->xEventListItem ), ( TickType_t ) configMAX_PRIORITIES - ( TickType_t ) uxPriorityToUse ); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + /* Disinherit the priority, remembering the previous + * priority to facilitate determining the subject task's + * state. */ + traceTASK_PRIORITY_DISINHERIT( pxTCB, uxPriorityToUse ); + uxPriorityUsedOnEntry = pxTCB->uxPriority; + pxTCB->uxPriority = uxPriorityToUse; - /* If the running task is not the task that holds the mutex - * then the task that holds the mutex could be in either the - * Ready, Blocked or Suspended states. Only remove the task - * from its current state list if it is in the Ready state as - * the task's priority is going to change and there is one - * Ready list per priority. */ - if( listIS_CONTAINED_WITHIN( &( pxReadyTasksLists[ uxPriorityUsedOnEntry ] ), &( pxTCB->xStateListItem ) ) != pdFALSE ) - { - if( uxListRemove( &( pxTCB->xStateListItem ) ) == ( UBaseType_t ) 0 ) + /* Only reset the event list item value if the value is not + * being used for anything else. */ + if( ( listGET_LIST_ITEM_VALUE( &( pxTCB->xEventListItem ) ) & taskEVENT_LIST_ITEM_VALUE_IN_USE ) == ( ( TickType_t ) 0U ) ) { - /* It is known that the task is in its ready list so - * there is no need to check again and the port level - * reset macro can be called directly. */ - portRESET_READY_PRIORITY( uxPriorityUsedOnEntry, uxTopReadyPriority ); + listSET_LIST_ITEM_VALUE( &( pxTCB->xEventListItem ), ( TickType_t ) configMAX_PRIORITIES - ( TickType_t ) uxPriorityToUse ); } else { mtCOVERAGE_TEST_MARKER(); } - prvAddTaskToReadyList( pxTCB ); - #if ( configNUMBER_OF_CORES > 1 ) + /* If the running task is not the task that holds the mutex + * then the task that holds the mutex could be in either the + * Ready, Blocked or Suspended states. Only remove the task + * from its current state list if it is in the Ready state as + * the task's priority is going to change and there is one + * Ready list per priority. */ + if( listIS_CONTAINED_WITHIN( &( pxReadyTasksLists[ uxPriorityUsedOnEntry ] ), &( pxTCB->xStateListItem ) ) != pdFALSE ) { - /* The priority of the task is dropped. Yield the core on - * which the task is running. */ - if( taskTASK_IS_RUNNING( pxTCB ) == pdTRUE ) + if( uxListRemove( &( pxTCB->xStateListItem ) ) == ( UBaseType_t ) 0 ) { - prvYieldCore( pxTCB->xTaskRunState ); + /* It is known that the task is in its ready list so + * there is no need to check again and the port level + * reset macro can be called directly. */ + portRESET_READY_PRIORITY( uxPriorityUsedOnEntry, uxTopReadyPriority ); } + else + { + mtCOVERAGE_TEST_MARKER(); + } + + prvAddTaskToReadyList( pxTCB ); + #if ( configNUMBER_OF_CORES > 1 ) + { + /* The priority of the task is dropped. Yield the core on + * which the task is running. */ + if( taskTASK_IS_RUNNING( pxTCB ) == pdTRUE ) + { + prvYieldCore( pxTCB->xTaskRunState ); + } + } + #endif /* if ( configNUMBER_OF_CORES > 1 ) */ + } + else + { + mtCOVERAGE_TEST_MARKER(); } - #endif /* if ( configNUMBER_OF_CORES > 1 ) */ } else { @@ -7003,10 +7405,9 @@ STATIC void prvResetNextTaskUnblockTime( void ) mtCOVERAGE_TEST_MARKER(); } } - else - { - mtCOVERAGE_TEST_MARKER(); - } + #if ( portUSING_GRANULAR_LOCKS == 1 ) + kernelEXIT_CRITICAL(); + #endif traceRETURN_vTaskPriorityDisinheritAfterTimeout(); } @@ -7081,7 +7482,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) #endif /* #if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) && ( configNUMBER_OF_CORES == 1 ) ) */ /*-----------------------------------------------------------*/ -#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) +#if ( configNUMBER_OF_CORES > 1 ) void vTaskEnterCritical( void ) { @@ -7093,11 +7494,24 @@ STATIC void prvResetNextTaskUnblockTime( void ) if( xSchedulerRunning != pdFALSE ) { - if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + /* When using granular locks, the critical section nesting count + * might have already been incremented if this call is a nested + * call from a data group critical section. Hence, we have to + * acquire the kernel task and ISR locks unconditionally. */ + #if ( portUSING_GRANULAR_LOCKS == 1 ) { - portGET_TASK_LOCK( xCoreID ); - portGET_ISR_LOCK( xCoreID ); + kernelGET_TASK_LOCK( xCoreID ); + kernelGET_ISR_LOCK( xCoreID ); } + #else /* portUSING_GRANULAR_LOCKS */ + { + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + { + kernelGET_TASK_LOCK( xCoreID ); + kernelGET_ISR_LOCK( xCoreID ); + } + } + #endif /* portUSING_GRANULAR_LOCKS */ portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); @@ -7130,10 +7544,10 @@ STATIC void prvResetNextTaskUnblockTime( void ) traceRETURN_vTaskEnterCritical(); } -#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ +#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ /*-----------------------------------------------------------*/ -#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) +#if ( configNUMBER_OF_CORES > 1 ) UBaseType_t vTaskEnterCriticalFromISR( void ) { @@ -7146,10 +7560,18 @@ STATIC void prvResetNextTaskUnblockTime( void ) { uxSavedInterruptStatus = portSET_INTERRUPT_MASK_FROM_ISR(); - if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + #if ( portUSING_GRANULAR_LOCKS == 1 ) { - portGET_ISR_LOCK( xCoreID ); + kernelGET_ISR_LOCK( xCoreID ); } + #else /* portUSING_GRANULAR_LOCKS */ + { + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + { + kernelGET_ISR_LOCK( xCoreID ); + } + } + #endif /* portUSING_GRANULAR_LOCKS */ portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); } @@ -7163,7 +7585,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) return uxSavedInterruptStatus; } -#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ +#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ /*-----------------------------------------------------------*/ #if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) && ( configNUMBER_OF_CORES == 1 ) ) @@ -7211,7 +7633,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) #endif /* #if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) && ( configNUMBER_OF_CORES == 1 ) ) */ /*-----------------------------------------------------------*/ -#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) +#if ( configNUMBER_OF_CORES > 1 ) void vTaskExitCritical( void ) { @@ -7231,32 +7653,79 @@ STATIC void prvResetNextTaskUnblockTime( void ) if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ) { - portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); - - if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + #if ( portUSING_GRANULAR_LOCKS == 1 ) { BaseType_t xYieldCurrentTask; - /* Get the xYieldPending stats inside the critical section. */ - xYieldCurrentTask = xYieldPendings[ xCoreID ]; - - portRELEASE_ISR_LOCK( xCoreID ); - portRELEASE_TASK_LOCK( xCoreID ); - portENABLE_INTERRUPTS(); - - /* When a task yields in a critical section it just sets - * xYieldPending to true. So now that we have exited the - * critical section check if xYieldPending is true, and - * if so yield. */ - if( xYieldCurrentTask != pdFALSE ) + /* Get the xYieldPending status inside the critical section. */ + if( ( xYieldPendings[ xCoreID ] == pdTRUE ) && ( uxSchedulerSuspended == pdFALSE ) + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + && ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) && + ( pxCurrentTCBs[ xCoreID ]->uxDeferredStateChange == 0U ) + #endif /* ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + ) { - portYIELD(); + xYieldCurrentTask = pdTRUE; + } + else + { + xYieldCurrentTask = pdFALSE; + } + + /* Release the ISR and task locks first when using granular locks. */ + kernelRELEASE_ISR_LOCK( xCoreID ); + kernelRELEASE_TASK_LOCK( xCoreID ); + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + { + portENABLE_INTERRUPTS(); + + /* When a task yields in a critical section it just sets + * xYieldPending to true. So now that we have exited the + * critical section check if xYieldPending is true, and + * if so yield. */ + if( xYieldCurrentTask != pdFALSE ) + { + portYIELD(); + } + } + else + { + mtCOVERAGE_TEST_MARKER(); } } - else + #else /* portUSING_GRANULAR_LOCKS */ { - mtCOVERAGE_TEST_MARKER(); + /* Decrement first; release locks and enable interrupts when count reaches zero. */ + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + { + BaseType_t xYieldCurrentTask; + + /* Get the xYieldPending stats inside the critical section. */ + xYieldCurrentTask = xYieldPendings[ xCoreID ]; + + kernelRELEASE_ISR_LOCK( xCoreID ); + kernelRELEASE_TASK_LOCK( xCoreID ); + portENABLE_INTERRUPTS(); + + /* When a task yields in a critical section it just sets + * xYieldPending to true. So now that we have exited the + * critical section check if xYieldPending is true, and + * if so yield. */ + if( xYieldCurrentTask != pdFALSE ) + { + portYIELD(); + } + } + else + { + mtCOVERAGE_TEST_MARKER(); + } } + #endif /* portUSING_GRANULAR_LOCKS */ } else { @@ -7271,10 +7740,10 @@ STATIC void prvResetNextTaskUnblockTime( void ) traceRETURN_vTaskExitCritical(); } -#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ +#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ /*-----------------------------------------------------------*/ -#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) +#if ( configNUMBER_OF_CORES > 1 ) void vTaskExitCriticalFromISR( UBaseType_t uxSavedInterruptStatus ) { @@ -7292,17 +7761,28 @@ STATIC void prvResetNextTaskUnblockTime( void ) if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ) { - portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + #if ( portUSING_GRANULAR_LOCKS == 1 ) + { + kernelRELEASE_ISR_LOCK( xCoreID ); - if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) - { - portRELEASE_ISR_LOCK( xCoreID ); - portCLEAR_INTERRUPT_MASK_FROM_ISR( uxSavedInterruptStatus ); + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + { + portCLEAR_INTERRUPT_MASK_FROM_ISR( uxSavedInterruptStatus ); + } } - else + #else /* portUSING_GRANULAR_LOCKS */ { - mtCOVERAGE_TEST_MARKER(); + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + { + kernelRELEASE_ISR_LOCK( xCoreID ); + portCLEAR_INTERRUPT_MASK_FROM_ISR( uxSavedInterruptStatus ); + } } + #endif /* portUSING_GRANULAR_LOCKS */ } else { @@ -7317,29 +7797,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) traceRETURN_vTaskExitCriticalFromISR(); } -#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */ -/*-----------------------------------------------------------*/ - -#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - - BaseType_t xTaskUnlockCanYield( void ) - { - BaseType_t xReturn; - BaseType_t xCoreID = portGET_CORE_ID(); - - if( ( xYieldPendings[ xCoreID ] == pdTRUE ) && ( uxSchedulerSuspended == pdFALSE ) && ( pxCurrentTCBs[ xCoreID ]->xPreemptionDisable == pdFALSE ) ) - { - xReturn = pdTRUE; - } - else - { - xReturn = pdFALSE; - } - - return xReturn; - } - -#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ +#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ /*-----------------------------------------------------------*/ #if ( configUSE_STATS_FORMATTING_FUNCTIONS > 0 ) @@ -7753,6 +8211,11 @@ TickType_t uxTaskResetEventItemValue( void ) traceENTER_pvTaskIncrementMutexHeldCount(); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* Lock the kernel data group as we are about to access its members */ + kernelENTER_CRITICAL(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + pxTCB = pxCurrentTCB; /* If xSemaphoreCreateMutex() is called before any tasks have been created @@ -7762,6 +8225,11 @@ TickType_t uxTaskResetEventItemValue( void ) ( pxTCB->uxMutexesHeld )++; } + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* We are done accessing the kernel data group. Unlock it. */ + kernelEXIT_CRITICAL(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceRETURN_pvTaskIncrementMutexHeldCount( pxTCB ); return pxTCB; @@ -7795,7 +8263,7 @@ TickType_t uxTaskResetEventItemValue( void ) * has occurred and set the flag to indicate that we are waiting for * a notification. If we do not do so, a notification sent from an ISR * will get lost. */ - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { /* Only block if the notification count is not already non-zero. */ if( pxCurrentTCB->ulNotifiedValue[ uxIndexToWaitOn ] == 0U ) @@ -7811,7 +8279,7 @@ TickType_t uxTaskResetEventItemValue( void ) mtCOVERAGE_TEST_MARKER(); } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); /* We are now out of the critical section but the scheduler is still * suspended, so we are safe to do non-deterministic operations such @@ -7839,7 +8307,7 @@ TickType_t uxTaskResetEventItemValue( void ) } } - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { traceTASK_NOTIFY_TAKE( uxIndexToWaitOn ); ulReturn = pxCurrentTCB->ulNotifiedValue[ uxIndexToWaitOn ]; @@ -7862,7 +8330,7 @@ TickType_t uxTaskResetEventItemValue( void ) pxCurrentTCB->ucNotifyState[ uxIndexToWaitOn ] = taskNOT_WAITING_NOTIFICATION; } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); traceRETURN_ulTaskGenericNotifyTake( ulReturn ); @@ -7897,7 +8365,7 @@ TickType_t uxTaskResetEventItemValue( void ) /* We MUST enter a critical section to atomically check and update the * task notification value. If we do not do so, a notification from * an ISR will get lost. */ - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { /* Only block if a notification is not already pending. */ if( pxCurrentTCB->ucNotifyState[ uxIndexToWaitOn ] != taskNOTIFICATION_RECEIVED ) @@ -7918,7 +8386,7 @@ TickType_t uxTaskResetEventItemValue( void ) mtCOVERAGE_TEST_MARKER(); } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); /* We are now out of the critical section but the scheduler is still * suspended, so we are safe to do non-deterministic operations such @@ -7946,7 +8414,7 @@ TickType_t uxTaskResetEventItemValue( void ) } } - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { traceTASK_NOTIFY_WAIT( uxIndexToWaitOn ); @@ -7976,7 +8444,7 @@ TickType_t uxTaskResetEventItemValue( void ) pxCurrentTCB->ucNotifyState[ uxIndexToWaitOn ] = taskNOT_WAITING_NOTIFICATION; } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); traceRETURN_xTaskGenericNotifyWait( xReturn ); @@ -8004,7 +8472,7 @@ TickType_t uxTaskResetEventItemValue( void ) configASSERT( xTaskToNotify ); pxTCB = xTaskToNotify; - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { if( pulPreviousNotificationValue != NULL ) { @@ -8096,7 +8564,7 @@ TickType_t uxTaskResetEventItemValue( void ) mtCOVERAGE_TEST_MARKER(); } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); traceRETURN_xTaskGenericNotify( xReturn ); @@ -8148,7 +8616,7 @@ TickType_t uxTaskResetEventItemValue( void ) /* MISRA Ref 4.7.1 [Return value shall be checked] */ /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#dir-47 */ /* coverity[misra_c_2012_directive_4_7_violation] */ - uxSavedInterruptStatus = ( UBaseType_t ) taskENTER_CRITICAL_FROM_ISR(); + uxSavedInterruptStatus = ( UBaseType_t ) kernelENTER_CRITICAL_FROM_ISR(); { if( pulPreviousNotificationValue != NULL ) { @@ -8278,7 +8746,7 @@ TickType_t uxTaskResetEventItemValue( void ) #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ } } - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); traceRETURN_xTaskGenericNotifyFromISR( xReturn ); @@ -8326,7 +8794,7 @@ TickType_t uxTaskResetEventItemValue( void ) /* MISRA Ref 4.7.1 [Return value shall be checked] */ /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#dir-47 */ /* coverity[misra_c_2012_directive_4_7_violation] */ - uxSavedInterruptStatus = ( UBaseType_t ) taskENTER_CRITICAL_FROM_ISR(); + uxSavedInterruptStatus = ( UBaseType_t ) kernelENTER_CRITICAL_FROM_ISR(); { ucOriginalNotifyState = pxTCB->ucNotifyState[ uxIndexToNotify ]; pxTCB->ucNotifyState[ uxIndexToNotify ] = taskNOTIFICATION_RECEIVED; @@ -8412,7 +8880,7 @@ TickType_t uxTaskResetEventItemValue( void ) #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ } } - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); traceRETURN_vTaskGenericNotifyGiveFromISR(); } @@ -8437,7 +8905,7 @@ TickType_t uxTaskResetEventItemValue( void ) pxTCB = prvGetTCBFromHandle( xTask ); configASSERT( pxTCB != NULL ); - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { if( pxTCB->ucNotifyState[ uxIndexToClear ] == taskNOTIFICATION_RECEIVED ) { @@ -8449,7 +8917,7 @@ TickType_t uxTaskResetEventItemValue( void ) xReturn = pdFAIL; } } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); traceRETURN_xTaskGenericNotifyStateClear( xReturn ); @@ -8477,14 +8945,14 @@ TickType_t uxTaskResetEventItemValue( void ) pxTCB = prvGetTCBFromHandle( xTask ); configASSERT( pxTCB != NULL ); - taskENTER_CRITICAL(); + kernelENTER_CRITICAL(); { /* Return the notification as it was before the bits were cleared, * then clear the bit mask. */ ulReturn = pxTCB->ulNotifiedValue[ uxIndexToClear ]; pxTCB->ulNotifiedValue[ uxIndexToClear ] &= ~ulBitsToClear; } - taskEXIT_CRITICAL(); + kernelEXIT_CRITICAL(); traceRETURN_ulTaskGenericNotifyValueClear( ulReturn ); From 1ae917325c593ee8aabccd30cf5ed72f7c8db8f2 Mon Sep 17 00:00:00 2001 From: Sudeep Mohanty Date: Sat, 6 Jun 2026 11:36:05 +0200 Subject: [PATCH 06/11] feat(freertos/smp): Add support for TCB locks --- event_groups.c | 58 ++- include/FreeRTOS.h | 7 +- include/task.h | 57 ++- queue.c | 87 +++- tasks.c | 1166 +++++++++++++++++++++++++++++++++----------- timers.c | 10 + 6 files changed, 1059 insertions(+), 326 deletions(-) diff --git a/event_groups.c b/event_groups.c index c69b96557..ceb3d0d75 100644 --- a/event_groups.c +++ b/event_groups.c @@ -81,6 +81,34 @@ /*-----------------------------------------------------------*/ +/* + * Macros used to lock and unlock an event group. When a task locks an, + * event group, the task will have thread safe non-deterministic access to + * the event group. + * - Concurrent access from other tasks will be blocked by the xTaskSpinlock + * - Concurrent access from ISRs will be pended + * + * When the task unlocks the event group, all pended access attempts are handled. + */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #define event_groupsLOCK( pxEventBits ) taskDATA_GROUP_LOCK( &( ( pxEventBits )->xTaskSpinlock ) ) + #define event_groupsUNLOCK( pxEventBits ) taskDATA_GROUP_UNLOCK( &( ( pxEventBits )->xTaskSpinlock ) ) + #define event_groupsUNLOCK_WITH_YIELD_STATUS( pxEventBits, pxAlreadyYielded ) \ + do { \ + *( pxAlreadyYielded ) = taskDATA_GROUP_UNLOCK( &( ( pxEventBits )->xTaskSpinlock ) ); \ + } while( 0 ) + #else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #define event_groupsLOCK( pxEventBits ) vTaskSuspendAll() + #define event_groupsUNLOCK( pxEventBits ) xTaskResumeAll() + #define event_groupsUNLOCK_WITH_YIELD_STATUS( pxEventBits, pxAlreadyYielded ) \ + do { \ + *( pxAlreadyYielded ) = xTaskResumeAll(); \ + } while( 0 ) + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + +/*-----------------------------------------------------------*/ + + #if ( configSUPPORT_STATIC_ALLOCATION == 1 ) EventGroupHandle_t xEventGroupCreateStatic( StaticEventGroup_t * pxEventGroupBuffer ) @@ -245,7 +273,7 @@ } } } - xAlreadyYielded = xTaskResumeAll(); + event_groupsUNLOCK_WITH_YIELD_STATUS( pxEventBits, &xAlreadyYielded ); if( xTicksToWait != ( TickType_t ) 0 ) { @@ -401,7 +429,7 @@ traceEVENT_GROUP_WAIT_BITS_BLOCK( xEventGroup, uxBitsToWaitFor ); } } - xAlreadyYielded = xTaskResumeAll(); + event_groupsUNLOCK_WITH_YIELD_STATUS( pxEventBits, &xAlreadyYielded ); if( xTicksToWait != ( TickType_t ) 0 ) { @@ -568,6 +596,13 @@ { traceEVENT_GROUP_SET_BITS( xEventGroup, uxBitsToSet ); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + + /* We are about to access the kernel data group non-deterministically, + * thus we suspend the kernel data group.*/ + vTaskSuspendAll(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + pxListItem = listGET_HEAD_ENTRY( pxList ); /* Set the bits. */ @@ -638,8 +673,12 @@ /* Snapshot resulting bits. */ uxReturnBits = pxEventBits->uxEventBits; + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + ( void ) xTaskResumeAll(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ } - ( void ) xTaskResumeAll(); + event_groupsUNLOCK( pxEventBits ); traceRETURN_xEventGroupSetBits( uxReturnBits ); @@ -662,6 +701,13 @@ { traceEVENT_GROUP_DELETE( xEventGroup ); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + + /* We are about to access the kernel data group non-deterministically, + * thus we suspend the kernel data group.*/ + vTaskSuspendAll(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + while( listCURRENT_LIST_LENGTH( pxTasksWaitingForBits ) > ( UBaseType_t ) 0 ) { /* Unblock the task, returning 0 as the event list is being deleted @@ -669,8 +715,12 @@ configASSERT( pxTasksWaitingForBits->xListEnd.pxNext != ( const ListItem_t * ) &( pxTasksWaitingForBits->xListEnd ) ); vTaskRemoveFromUnorderedEventList( pxTasksWaitingForBits->xListEnd.pxNext, eventUNBLOCKED_DUE_TO_BIT_SET ); } + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + ( void ) xTaskResumeAll(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ } - ( void ) xTaskResumeAll(); + event_groupsUNLOCK( pxEventBits ); #if ( ( configSUPPORT_DYNAMIC_ALLOCATION == 1 ) && ( configSUPPORT_STATIC_ALLOCATION == 0 ) ) { diff --git a/include/FreeRTOS.h b/include/FreeRTOS.h index e3f3bb300..dc2748d97 100644 --- a/include/FreeRTOS.h +++ b/include/FreeRTOS.h @@ -2942,10 +2942,6 @@ #error configUSE_MUTEXES must be set to 1 to use recursive mutexes #endif -#if ( ( configRUN_MULTIPLE_PRIORITIES == 0 ) && ( configUSE_TASK_PREEMPTION_DISABLE != 0 ) ) - #error configRUN_MULTIPLE_PRIORITIES must be set to 1 to use task preemption disable -#endif - #if ( ( configUSE_PREEMPTION == 0 ) && ( configUSE_TASK_PREEMPTION_DISABLE != 0 ) ) #error configUSE_PREEMPTION must be set to 1 to use task preemption disable #endif @@ -3245,6 +3241,9 @@ typedef struct xSTATIC_TCB #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) BaseType_t xDummy26; #endif + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portSPINLOCK_TYPE xDummy27; + #endif #if ( ( portSTACK_GROWTH > 0 ) || ( configRECORD_STACK_HIGH_ADDRESS == 1 ) ) void * pxDummy8; #endif diff --git a/include/task.h b/include/task.h index 57da38118..5781e17e6 100644 --- a/include/task.h +++ b/include/task.h @@ -296,22 +296,12 @@ typedef enum * \ingroup GranularLocks */ #if ( portUSING_GRANULAR_LOCKS == 1 ) - #define taskDATA_GROUP_ENTER_CRITICAL( pxTaskSpinlock, pxISRSpinlock ) \ - do { \ - /* Disable preemption to avoid task state changes during the critical section. */ \ - vTaskPreemptionDisable( NULL ); \ - { \ - const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ - /* Task spinlock is always taken first */ \ - portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ); \ - /* Disable interrupts */ \ - portDISABLE_INTERRUPTS(); \ - /* Take the ISR spinlock next */ \ - portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) ( pxISRSpinlock ) ); \ - /* Increment the critical nesting count */ \ - portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ - } \ - } while( 0 ) + +/* Using a function implementation now since the data group entering critical + * section needs to check for run state change. */ + void taskDataGroupEnterCritical( portSPINLOCK_TYPE * pxTaskSpinlock, + portSPINLOCK_TYPE * pxISRSpinlock ); + #define taskDATA_GROUP_ENTER_CRITICAL taskDataGroupEnterCritical #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ /** @@ -365,7 +355,7 @@ typedef enum mtCOVERAGE_TEST_MARKER(); \ } \ /* Re-enable preemption */ \ - prvTaskPreemptionEnable( NULL ); \ + ( void ) xTaskPreemptionEnableWithYieldStatus( NULL ); \ } while( 0 ) #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ @@ -421,12 +411,12 @@ typedef enum * \ingroup GranularLocks */ #if ( portUSING_GRANULAR_LOCKS == 1 ) - #define taskDATA_GROUP_UNLOCK( pxTaskSpinlock ) \ - ( { \ - portRELEASE_SPINLOCK( portGET_CORE_ID(), ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ); \ - /* Re-enable preemption after releasing the task spinlock. */ \ - prvTaskPreemptionEnable( NULL ); \ - } ) + +/* Release the task spinlock and re-enable preemption. + * Returns the yield status reported by xTaskPreemptionEnableWithYieldStatus(). */ + #define taskDATA_GROUP_UNLOCK( pxTaskSpinlock ) \ + ( portRELEASE_SPINLOCK( portGET_CORE_ID(), ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ), \ + xTaskPreemptionEnableWithYieldStatus( NULL ) ) #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ /*----------------------------------------------------------- @@ -1644,7 +1634,7 @@ BaseType_t xTaskResumeFromISR( TaskHandle_t xTaskToResume ) PRIVILEGED_FUNCTION; * switch, otherwise pdFALSE. This is used by the scheduler to determine if a * context switch may be required following the enable. */ - BaseType_t prvTaskPreemptionEnable( const TaskHandle_t xTask ); + BaseType_t xTaskPreemptionEnableWithYieldStatus( const TaskHandle_t xTask ); #endif /*----------------------------------------------------------- @@ -3727,6 +3717,8 @@ void vTaskPlaceOnEventListRestricted( List_t * const pxEventList, * Removes a task from both the specified event list and the list of blocked * tasks, and places it on a ready queue. * + * Do not call this function from an ISR context. Call xTaskRemoveFromEventListFromISR() instead. + * * xTaskRemoveFromEventList()/vTaskRemoveFromUnorderedEventList() will be called * if either an event occurs to unblock a task, or the block timeout period * expires. @@ -3743,6 +3735,23 @@ void vTaskPlaceOnEventListRestricted( List_t * const pxEventList, * making the call, otherwise pdFALSE. */ BaseType_t xTaskRemoveFromEventList( const List_t * const pxEventList ) PRIVILEGED_FUNCTION; + +/* + * THIS FUNCTION MUST NOT BE USED FROM APPLICATION CODE. IT IS ONLY + * INTENDED FOR USE WHEN IMPLEMENTING A PORT OF THE SCHEDULER AND IS + * AN INTERFACE WHICH IS FOR THE EXCLUSIVE USE OF THE SCHEDULER. + * + * THIS FUNCTION MUST BE CALLED WITH INTERRUPTS DISABLED. + * + * Removes a task from both the specified event list and the list of blocked + * tasks, and places it on a ready queue. This function is the ISR-safe version + * of xTaskRemoveFromEventList(). + * + * @return pdTRUE if the task being removed has a higher priority than the task + * making the call, otherwise pdFALSE. + */ +BaseType_t xTaskRemoveFromEventListFromISR( const List_t * const pxEventList ) PRIVILEGED_FUNCTION; + void vTaskRemoveFromUnorderedEventList( ListItem_t * pxEventListItem, const TickType_t xItemValue ) PRIVILEGED_FUNCTION; diff --git a/queue.c b/queue.c index 8df1481e8..f8198348d 100644 --- a/queue.c +++ b/queue.c @@ -2498,7 +2498,15 @@ static void prvUnlockQueue( Queue_t * const pxQueue ) * removed from the queue while the queue was locked. When a queue is * locked items can be added or removed, but the event lists cannot be * updated. */ - taskENTER_CRITICAL(); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + const BaseType_t xCoreID = portGET_CORE_ID(); + + portDISABLE_INTERRUPTS(); + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->xISRSpinlock ) ); + #else + queueENTER_CRITICAL( pxQueue ); + #endif { int8_t cTxLock = pxQueue->cTxLock; @@ -2576,10 +2584,26 @@ static void prvUnlockQueue( Queue_t * const pxQueue ) pxQueue->cTxLock = queueUNLOCKED; } - taskEXIT_CRITICAL(); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->xISRSpinlock ) ); + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + { + portENABLE_INTERRUPTS(); + } + #else + queueEXIT_CRITICAL( pxQueue ); + #endif /* Do the same for the Rx lock. */ - taskENTER_CRITICAL(); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + portDISABLE_INTERRUPTS(); + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->xISRSpinlock ) ); + #else + queueENTER_CRITICAL( pxQueue ); + #endif { int8_t cRxLock = pxQueue->cRxLock; @@ -3325,8 +3349,63 @@ BaseType_t xQueueIsQueueFullFromISR( const QueueHandle_t xQueue ) /*-----------------------------------------------------------*/ #if ( configUSE_QUEUE_SETS == 1 ) - static BaseType_t prvNotifyQueueSetContainer( const Queue_t * const pxQueue ) + { + BaseType_t xReturn; + + /* Call the generic version with xIsISR = pdFALSE to indicate task context */ + + #if ( portUSING_GRANULAR_LOCKS == 0 ) + { + xReturn = prvNotifyQueueSetContainerGeneric( pxQueue, pdFALSE ); + } + #else + { + const BaseType_t xCoreID = portGET_CORE_ID(); + + /* This API must be called in a critical section which already has preemption + * and interrupt disabled. */ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xTaskSpinlock ) ); + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xISRSpinlock ) ); + { + xReturn = prvNotifyQueueSetContainerGeneric( pxQueue, pdFALSE ); + } + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xISRSpinlock ) ); + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xTaskSpinlock ) ); + } + #endif /* if ( portUSING_GRANULAR_LOCKS == 0 ) */ + + return xReturn; + } + + static BaseType_t prvNotifyQueueSetContainerFromISR( const Queue_t * const pxQueue ) + { + BaseType_t xReturn; + + /* Call the generic version with xIsISR = pdTRUE to indicate ISR context */ + + #if ( portUSING_GRANULAR_LOCKS == 0 ) + { + xReturn = prvNotifyQueueSetContainerGeneric( pxQueue, pdTRUE ); + } + #else + { + const BaseType_t xCoreID = portGET_CORE_ID(); + + /* This API must be called in a critical section which already has interrupt disabled. */ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xISRSpinlock ) ); + { + xReturn = prvNotifyQueueSetContainerGeneric( pxQueue, pdTRUE ); + } + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xISRSpinlock ) ); + } + #endif /* if ( portUSING_GRANULAR_LOCKS == 0 ) */ + + return xReturn; + } + + static BaseType_t prvNotifyQueueSetContainerGeneric( const Queue_t * const pxQueue, + const BaseType_t xIsISR ) { Queue_t * pxQueueSetContainer = pxQueue->pxQueueSetContainer; BaseType_t xReturn = pdFALSE; diff --git a/tasks.c b/tasks.c index 8401f2cef..68caa2228 100644 --- a/tasks.c +++ b/tasks.c @@ -354,7 +354,64 @@ * must be valid. This macro is not required in single core since there is only * one core to yield. */ #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - #define prvYieldCore( xCoreID ) \ + #if ( portUSING_GRANULAR_LOCKS == 1 ) + +/* xYieldPending indicates that a yield was requested but could not be + * processed immediately. + * + * This occurs in situations such as: + * 1. Task preemption is disabled. + * 2. A task requests itself to yield while inside a critical section. + * + * In these cases, xYieldPending is set to pdTRUE to indicate that the + * yield request should be serviced after exiting the critical section. + * + * With granular locking, the task disables preemption first before acquiring + * data group locks. A check for run state change is required before acquiring + * the data group locks to preserve priority ordering, for example, in queue + * operations. However, setting xYieldPending alone is not sufficient. Without + * setting the task run state to taskTASK_SCHEDULED_TO_YIELD, the task would + * not be able to relinquish the critical section for higher priority tasks. + * + * Key distinction: + * - xYieldPending: yield is serviced when leaving the critical section. + * - taskTASK_SCHEDULED_TO_YIELD: yield is serviced before entering the + * critical section. */ + #define prvYieldCore( xCoreID ) \ + do { \ + const BaseType_t xCoreToYield = ( xCoreID ); \ + BaseType_t xCurrentCoreID = portGET_CORE_ID(); \ + if( ( xCoreID ) == xCurrentCoreID ) \ + { \ + /* Pending a yield for this core since it is in the critical section. */ \ + xYieldPendings[ xCoreToYield ] = pdTRUE; \ + pxCurrentTCBs[ xCoreToYield ]->xTaskRunState = taskTASK_SCHEDULED_TO_YIELD; \ + } \ + else \ + { \ + portGET_SPINLOCK( xCurrentCoreID, &( pxCurrentTCBs[ xCoreToYield ]->xTCBSpinlock ) ); \ + { \ + if( pxCurrentTCBs[ xCoreToYield ]->uxPreemptionDisable == 0U ) \ + { \ + /* Request other core to yield if it is not requested before. */ \ + if( pxCurrentTCBs[ xCoreToYield ]->xTaskRunState != taskTASK_SCHEDULED_TO_YIELD ) \ + { \ + portYIELD_CORE( xCoreToYield ); \ + pxCurrentTCBs[ xCoreToYield ]->xTaskRunState = taskTASK_SCHEDULED_TO_YIELD; \ + } \ + } \ + else \ + { \ + xYieldPendings[ xCoreToYield ] = pdTRUE; \ + pxCurrentTCBs[ xCoreToYield ]->xTaskRunState = taskTASK_SCHEDULED_TO_YIELD; \ + } \ + } \ + portRELEASE_SPINLOCK( xCurrentCoreID, &( pxCurrentTCBs[ xCoreToYield ]->xTCBSpinlock ) ); \ + } \ + } while( 0 ) + + #else /* if ( portUSING_GRANULAR_LOCKS == 1 ) */ + #define prvYieldCore( xCoreID ) \ do { \ if( ( xCoreID ) == ( BaseType_t ) portGET_CORE_ID() ) \ { \ @@ -378,6 +435,7 @@ } \ } \ } while( 0 ) + #endif /* if ( portUSING_GRANULAR_LOCKS == 1 ) */ #else /* if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ #define prvYieldCore( xCoreID ) \ do { \ @@ -430,6 +488,20 @@ #define kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ +/* + * Helper macros that pick the appropriate critical-section primitive based on + * the number of cores configured. On SMP builds, multi-core access requires + * the kernel data-group lock; on single-core builds, the lighter + * port-provided primitive is sufficient. + */ +#if ( configNUMBER_OF_CORES > 1 ) + #define taskBASE_TYPE_ENTER_CRITICAL() kernelENTER_CRITICAL() + #define taskBASE_TYPE_EXIT_CRITICAL() kernelEXIT_CRITICAL() +#else /* #if ( configNUMBER_OF_CORES > 1 ) */ + #define taskBASE_TYPE_ENTER_CRITICAL() portBASE_TYPE_ENTER_CRITICAL() + #define taskBASE_TYPE_EXIT_CRITICAL() portBASE_TYPE_EXIT_CRITICAL() +#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ + /*-----------------------------------------------------------*/ /* @@ -467,6 +539,10 @@ typedef struct tskTaskControlBlock /* The old naming convention is used to UBaseType_t uxDeferredStateChange; /**< Used to indicate if the task's state change is deferred. */ #endif + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portSPINLOCK_TYPE xTCBSpinlock; + #endif + #if ( ( portSTACK_GROWTH > 0 ) || ( configRECORD_STACK_HIGH_ADDRESS == 1 ) ) StackType_t * pxEndOfStack; /**< Points to the highest valid address for the stack. */ #endif @@ -709,6 +785,12 @@ STATIC portTASK_FUNCTION_PROTO( prvIdleTask, */ STATIC void prvCheckTasksWaitingTermination( void ) PRIVILEGED_FUNCTION; +/* + * Private helper function to remove a task from an event list. This function + * is shared between the task context and ISR context versions. + */ +static BaseType_t prvTaskRemoveFromEventList( const List_t * const pxEventList ) PRIVILEGED_FUNCTION; + /* * The currently executing task is entering the Blocked state. Add the task to * either the current or the overflow delayed task list. @@ -854,10 +936,10 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; #endif /* #if ( configSUPPORT_DYNAMIC_ALLOCATION == 1 ) */ /* - * Helper function to enable preemption for a task. + * Helper function to enable preemption for a task and return yield status. */ #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - BaseType_t prvTaskPreemptionEnable( const TaskHandle_t xTask ) PRIVILEGED_FUNCTION; + BaseType_t xTaskPreemptionEnableWithYieldStatus( const TaskHandle_t xTask ) PRIVILEGED_FUNCTION; #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ /* @@ -901,6 +983,23 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; size_t n ); #endif /* #if ( ( configUSE_TRACE_FACILITY == 1 ) && ( configUSE_STATS_FORMATTING_FUNCTIONS > 0 ) ) */ + +/* + * Helpers used to enter and exit the kernel critical section from a context + * that is already inside a data-group critical section (and therefore already + * holds the data-group's ISR spinlock with interrupts disabled). + * + * Unlike kernelENTER_CRITICAL_FROM_ISR/kernelEXIT_CRITICAL_FROM_ISR, these + * helpers do not touch the interrupt state: the outer data-group critical + * section is responsible for that. They only acquire/release the kernel + * ISR spinlock and adjust the per-core kernel critical-nesting count, so + * that callers can safely access kernel-data-group members while remaining + * within the surrounding data-group critical section's lock hierarchy. + */ +static void prvKernelEnterISROnlyCritical( void ); + +static void prvKernelExitISROnlyCritical( void ); + /*-----------------------------------------------------------*/ #if ( configNUMBER_OF_CORES > 1 ) @@ -929,20 +1028,11 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; * and reacquire the correct locks. And then, do it all over again * if our state changed again during the reacquisition. */ uxPrevCriticalNesting = portGET_CRITICAL_NESTING_COUNT( xCoreID ); + portSET_CRITICAL_NESTING_COUNT( xCoreID, 0U ); - if( uxPrevCriticalNesting > 0U ) - { - portSET_CRITICAL_NESTING_COUNT( xCoreID, 0U ); - kernelRELEASE_ISR_LOCK( xCoreID ); - } - else - { - /* The scheduler is suspended. uxSchedulerSuspended is updated - * only when the task is not requested to yield. */ - mtCOVERAGE_TEST_MARKER(); - } - + kernelRELEASE_ISR_LOCK( xCoreID ); kernelRELEASE_TASK_LOCK( xCoreID ); + portMEMORY_BARRIER(); configASSERT( pxThisTCB->xTaskRunState == taskTASK_SCHEDULED_TO_YIELD ); @@ -957,15 +1047,11 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; portDISABLE_INTERRUPTS(); xCoreID = ( BaseType_t ) portGET_CORE_ID(); + kernelGET_TASK_LOCK( xCoreID ); kernelGET_ISR_LOCK( xCoreID ); portSET_CRITICAL_NESTING_COUNT( xCoreID, uxPrevCriticalNesting ); - - if( uxPrevCriticalNesting == 0U ) - { - kernelRELEASE_ISR_LOCK( xCoreID ); - } } } #endif /* #if ( configNUMBER_OF_CORES > 1 ) */ @@ -1028,14 +1114,30 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; { #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) { + /* Acquire the preemption lock to prevent the task changes + * it's preemption state. */ + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portGET_SPINLOCK( xCurrentCoreID, &( pxCurrentTCBs[ xCoreID ]->xTCBSpinlock ) ); + #endif + if( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) { + #if ( portUSING_GRANULAR_LOCKS == 1 ) + if( xLowestPriorityCore != -1 ) + { + /* Release previous preemption lock. */ + portRELEASE_SPINLOCK( xCurrentCoreID, &( pxCurrentTCBs[ xLowestPriorityCore ]->xTCBSpinlock ) ); + } + #endif xLowestPriorityToPreempt = xCurrentCoreTaskPriority; xLowestPriorityCore = xCoreID; } else { xYieldPendings[ xCoreID ] = pdTRUE; + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portRELEASE_SPINLOCK( xCurrentCoreID, &( pxCurrentTCBs[ xCoreID ]->xTCBSpinlock ) ); + #endif } } #else /* if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ @@ -1084,6 +1186,18 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; prvYieldCore( xLowestPriorityCore ); } + /* After request the core to yield, preemption lock can be released. + * The task tries to set the preemption disable count will check it's + * run state first. If it is already requested to yield, it will serve + * the yield request first then retring to set the preemption disable + * count. */ + #if ( portUSING_GRANULAR_LOCKS == 1 ) + if( xLowestPriorityCore >= 0 ) + { + portRELEASE_SPINLOCK( xCurrentCoreID, &( pxCurrentTCBs[ xLowestPriorityCore ]->xTCBSpinlock ) ); + } + #endif + #if ( configRUN_MULTIPLE_PRIORITIES == 0 ) /* Verify that the calling core always yields to higher priority tasks. */ if( ( ( pxCurrentTCBs[ xCurrentCoreID ]->uxTaskAttributes & taskATTRIBUTE_IS_IDLE ) == 0U ) && @@ -1264,9 +1378,13 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; for( x = ( BaseType_t ) 0; x < ( BaseType_t ) configNUMBER_OF_CORES; x++ ) { - if( ( pxCurrentTCBs[ x ]->uxTaskAttributes & taskATTRIBUTE_IS_IDLE ) != 0U ) + /* Core xCoreID already selects the task to run. */ + if( x != xCoreID ) { - prvYieldCore( x ); + if( ( pxCurrentTCBs[ x ]->uxTaskAttributes & taskATTRIBUTE_IS_IDLE ) != 0U ) + { + prvYieldCore( x ); + } } } } @@ -1344,17 +1462,28 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; ( xYieldPendings[ uxCore ] == pdFALSE ) ) { #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - { + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portGET_SPINLOCK( xCoreID, &( pxCurrentTCBs[ uxCore ]->xTCBSpinlock ) ); + #endif + if( pxCurrentTCBs[ uxCore ]->uxPreemptionDisable == 0U ) { + #if ( portUSING_GRANULAR_LOCKS == 1 ) + if( xLowestPriorityCore != -1 ) + { + portRELEASE_SPINLOCK( xCoreID, &( pxCurrentTCBs[ xLowestPriorityCore ]->xTCBSpinlock ) ); + } + #endif xLowestPriority = xTaskPriority; xLowestPriorityCore = ( BaseType_t ) uxCore; } else { xYieldPendings[ uxCore ] = pdTRUE; + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portRELEASE_SPINLOCK( xCoreID, &( pxCurrentTCBs[ uxCore ]->xTCBSpinlock ) ); + #endif } - } #else /* if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ { xLowestPriority = xTaskPriority; @@ -1368,6 +1497,10 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; if( xLowestPriorityCore >= 0 ) { prvYieldCore( xLowestPriorityCore ); + + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portRELEASE_SPINLOCK( xCoreID, &( pxCurrentTCBs[ xLowestPriorityCore ]->xTCBSpinlock ) ); + #endif } } } @@ -2066,6 +2199,12 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, } #endif + #if ( portUSING_GRANULAR_LOCKS == 1 ) + { + portINIT_SPINLOCK( &pxNewTCB->xTCBSpinlock ); + } + #endif + /* Initialize the TCB stack to look as if the task was already running, * but had been interrupted by the scheduler. The return address is set * to the start of the task function. Once the stack has been initialised @@ -2342,25 +2481,45 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, kernelENTER_CRITICAL(); { + #if ( ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) ) + const BaseType_t xCoreID = portGET_CORE_ID(); + #endif /* ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) */ + /* If null is passed in here then it is the calling task that is * being deleted. */ pxTCB = prvGetTCBFromHandle( xTaskToDelete ); configASSERT( pxTCB != NULL ); - #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - - /* If the task has disabled preemption, we need to defer the deletion until the - * task enables preemption. The deletion will be performed in vTaskPreemptionEnable(). */ - if( pxTCB->uxPreemptionDisable > 0U ) + #if ( ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) ) + { + /* Acquire the TCB lock before reading the preemption-disable + * count to avoid a race with the target task running on another + * core (which may be modifying uxPreemptionDisable concurrently). */ + portGET_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); { - pxTCB->uxDeferredStateChange |= tskDEFERRED_DELETION; - xDeferredDeletion = pdTRUE; + /* If the task has disabled preemption, we need to defer the deletion until the + * task enables preemption. The deletion will be performed in vTaskPreemptionEnable(). */ + if( pxTCB->uxPreemptionDisable > 0U ) + { + /* Deletion takes priority over any other deferred action. + * Overwrite the flag word so that a previously-queued + * tskDEFERRED_SUSPENSION cannot be dispatched after this point. + * tskDEFERRED_DELETION will be dispatched in vTaskPreemptionEnable(). */ + pxTCB->uxDeferredStateChange = tskDEFERRED_DELETION; + xDeferredDeletion = pdTRUE; + portRELEASE_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); + } + else + { + /* Reset all deferred state change flags since the task is + * being deleted now. Any pending tskDEFERRED_SUSPENSION is irrelevant. + * Failing to clear it would leave the tskDEFERRED_SUSPENSION latched + * after the deletion completes, preventing the calling context from yielding. */ + pxTCB->uxDeferredStateChange = 0U; + } } - else - { - mtCOVERAGE_TEST_MARKER(); - } - #endif /* configUSE_TASK_PREEMPTION_DISABLE */ + } + #endif /* ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) */ #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) if( xDeferredDeletion == pdFALSE ) @@ -2466,6 +2625,10 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, * the task that has just been deleted. */ prvResetNextTaskUnblockTime(); } + + #if ( ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) ) + portRELEASE_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); + #endif } } kernelEXIT_CRITICAL(); @@ -2791,15 +2954,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, traceENTER_uxTaskPriorityGet( xTask ); - #if ( ( configNUMBER_OF_CORES > 1 ) ) - { - kernelENTER_CRITICAL(); - } - #else - { - portBASE_TYPE_ENTER_CRITICAL(); - } - #endif + taskBASE_TYPE_ENTER_CRITICAL(); { /* If null is passed in here then it is the priority of the task * that called uxTaskPriorityGet() that is being queried. */ @@ -2808,15 +2963,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, uxReturn = pxTCB->uxPriority; } - #if ( ( configNUMBER_OF_CORES > 1 ) ) - { - kernelEXIT_CRITICAL(); - } - #else - { - portBASE_TYPE_EXIT_CRITICAL(); - } - #endif + taskBASE_TYPE_EXIT_CRITICAL(); traceRETURN_uxTaskPriorityGet( uxReturn ); @@ -2885,15 +3032,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, traceENTER_uxTaskBasePriorityGet( xTask ); - #if ( ( configNUMBER_OF_CORES > 1 ) ) - { - kernelENTER_CRITICAL(); - } - #else - { - portBASE_TYPE_ENTER_CRITICAL(); - } - #endif + taskBASE_TYPE_ENTER_CRITICAL(); { /* If null is passed in here then it is the base priority of the task * that called uxTaskBasePriorityGet() that is being queried. */ @@ -2902,15 +3041,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, uxReturn = pxTCB->uxBasePriority; } - #if ( ( configNUMBER_OF_CORES > 1 ) ) - { - kernelEXIT_CRITICAL(); - } - #else - { - portBASE_TYPE_EXIT_CRITICAL(); - } - #endif + taskBASE_TYPE_EXIT_CRITICAL(); traceRETURN_uxTaskBasePriorityGet( uxReturn ); @@ -3245,30 +3376,14 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, traceENTER_vTaskCoreAffinityGet( xTask ); - #if ( ( configNUMBER_OF_CORES > 1 ) ) - { - kernelENTER_CRITICAL(); - } - #else - { - portBASE_TYPE_ENTER_CRITICAL(); - } - #endif + taskBASE_TYPE_ENTER_CRITICAL(); { pxTCB = prvGetTCBFromHandle( xTask ); configASSERT( pxTCB != NULL ); uxCoreAffinityMask = pxTCB->uxCoreAffinityMask; } - #if ( ( configNUMBER_OF_CORES > 1 ) ) - { - kernelEXIT_CRITICAL(); - } - #else - { - portBASE_TYPE_EXIT_CRITICAL(); - } - #endif + taskBASE_TYPE_EXIT_CRITICAL(); traceRETURN_vTaskCoreAffinityGet( uxCoreAffinityMask ); @@ -3278,15 +3393,232 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, /*-----------------------------------------------------------*/ -#if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) +#if ( portUSING_GRANULAR_LOCKS == 1 ) + static void prvTaskTCBLockCheckForRunStateChange( void ) + { + const TCB_t * pxThisTCB; + BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + + /* This must only be called from within a task. */ + portASSERT_IF_IN_ISR(); + + /* This function is always called with interrupts disabled + * so this is safe. */ + pxThisTCB = pxCurrentTCBs[ xCoreID ]; + + while( pxThisTCB->xTaskRunState == taskTASK_SCHEDULED_TO_YIELD ) + { + UBaseType_t uxPrevCriticalNesting; + + /* We are only here if we just entered a critical section + * or if we just suspended the scheduler, and another task + * has requested that we yield. + * + * This is slightly complicated since we need to save and restore + * the suspension and critical nesting counts, as well as release + * and reacquire the correct locks. And then, do it all over again + * if our state changed again during the reacquisition. */ + uxPrevCriticalNesting = portGET_CRITICAL_NESTING_COUNT( xCoreID ); + + if( uxPrevCriticalNesting > 0U ) + { + portSET_CRITICAL_NESTING_COUNT( xCoreID, 0U ); + portRELEASE_SPINLOCK( xCoreID, &pxCurrentTCBs[ xCoreID ]->xTCBSpinlock ); + } + else + { + /* The scheduler is suspended. uxSchedulerSuspended is updated + * only when the task is not requested to yield. */ + mtCOVERAGE_TEST_MARKER(); + } + + portMEMORY_BARRIER(); + + portENABLE_INTERRUPTS(); + + /* Enabling interrupts should cause this core to immediately service + * the pending interrupt and yield. After servicing the pending interrupt, + * the task needs to re-evaluate its run state within this loop, as + * other cores may have requested this task to yield, potentially altering + * its run state. */ + + portDISABLE_INTERRUPTS(); + + xCoreID = ( BaseType_t ) portGET_CORE_ID(); + portGET_SPINLOCK( xCoreID, &pxCurrentTCBs[ xCoreID ]->xTCBSpinlock ); + + portSET_CRITICAL_NESTING_COUNT( xCoreID, uxPrevCriticalNesting ); + } + } + + void vTaskTCBEnterCritical( void ) + { + if( xSchedulerRunning != pdFALSE ) + { + portDISABLE_INTERRUPTS(); + { + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + + portGET_SPINLOCK( xCoreID, &pxCurrentTCBs[ xCoreID ]->xTCBSpinlock ); + + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + + if( ( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 1U ) && + ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) && + ( uxSchedulerSuspended == 0U ) ) + { + prvTaskTCBLockCheckForRunStateChange(); + } + } + } + } + + void vTaskTCBExitCritical( void ) + { + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + + if( xSchedulerRunning != pdFALSE ) + { + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ) + { + BaseType_t xYieldCurrentTask = pdFALSE; + + /* Get the xYieldPending stats inside the critical section. */ + if( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) + { + xYieldCurrentTask = xYieldPendings[ xCoreID ]; + } + + portRELEASE_SPINLOCK( xCoreID, &pxCurrentTCBs[ xCoreID ]->xTCBSpinlock ); + + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + + /* If the critical nesting count is 0, enable interrupts */ + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + { + portENABLE_INTERRUPTS(); + + if( xYieldCurrentTask != pdFALSE ) + { + portYIELD(); + } + } + } + } + } + + static void prvTaskDataGroupCheckForRunStateChange( portSPINLOCK_TYPE * pxTaskSpinlock, + portSPINLOCK_TYPE * pxISRSpinlock ) + { + const TCB_t * pxThisTCB; + BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + + /* This must only be called from within a task. */ + portASSERT_IF_IN_ISR(); + + /* This function is always called with interrupts disabled + * so this is safe. */ + pxThisTCB = pxCurrentTCBs[ xCoreID ]; + + while( pxThisTCB->xTaskRunState == taskTASK_SCHEDULED_TO_YIELD ) + { + UBaseType_t uxPrevCriticalNesting; + + /* We are only here if we just entered a critical section + * or if we just suspended the scheduler, and another task + * has requested that we yield. + * + * This is slightly complicated since we need to save and restore + * the suspension and critical nesting counts, as well as release + * and reacquire the correct locks. And then, do it all over again + * if our state changed again during the reacquisition. */ + uxPrevCriticalNesting = portGET_CRITICAL_NESTING_COUNT( xCoreID ); + + /* Data group lock. */ + if( uxPrevCriticalNesting > 0U ) + { + portSET_CRITICAL_NESTING_COUNT( xCoreID, 0U ); + portRELEASE_SPINLOCK( xCoreID, pxISRSpinlock ); + } + else + { + /* The scheduler is suspended. uxSchedulerSuspended is updated + * only when the task is not requested to yield. */ + mtCOVERAGE_TEST_MARKER(); + } + + portENABLE_INTERRUPTS(); + + portRELEASE_SPINLOCK( xCoreID, pxTaskSpinlock ); + + portMEMORY_BARRIER(); + + xTaskPreemptionEnableWithYieldStatus( NULL ); + + + + /* Enabling interrupts should cause this core to immediately service + * the pending interrupt and yield. After servicing the pending interrupt, + * the task needs to re-evaluate its run state within this loop, as + * other cores may have requested this task to yield, potentially altering + * its run state. */ + + vTaskPreemptionDisable( NULL ); + + xCoreID = ( BaseType_t ) portGET_CORE_ID(); + portGET_SPINLOCK( xCoreID, pxTaskSpinlock ); + /* Disable interrupts */ + portDISABLE_INTERRUPTS(); + /* Take the ISR spinlock next */ + portGET_SPINLOCK( xCoreID, pxISRSpinlock ); + /* Increment the critical nesting count */ + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + } + } + + void taskDataGroupEnterCritical( portSPINLOCK_TYPE * pxTaskSpinlock, + portSPINLOCK_TYPE * pxISRSpinlock ) + { + /* Disable preemption to avoid task state changes during the critical section. */ + vTaskPreemptionDisable( NULL ); + { + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + /* Task spinlock is always taken first */ + portGET_SPINLOCK( xCoreID, pxTaskSpinlock ); + /* Disable interrupts */ + portDISABLE_INTERRUPTS(); + /* Take the ISR spinlock next */ + portGET_SPINLOCK( xCoreID, pxISRSpinlock ); + /* Increment the critical nesting count */ + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + + if( xSchedulerRunning == pdTRUE ) + { + if( ( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 1U ) && + ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 1U ) && + ( uxSchedulerSuspended == 0U ) ) + { + prvTaskDataGroupCheckForRunStateChange( pxTaskSpinlock, pxISRSpinlock ); + } + } + } + } +#endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ +/*-----------------------------------------------------------*/ + +#if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) void vTaskPreemptionDisable( const TaskHandle_t xTask ) { TCB_t * pxTCB; traceENTER_vTaskPreemptionDisable( xTask ); - kernelENTER_CRITICAL(); + #if ( portUSING_GRANULAR_LOCKS == 1 ) + vTaskTCBEnterCritical(); + #else + kernelENTER_CRITICAL(); + #endif { if( xSchedulerRunning != pdFALSE ) { @@ -3300,7 +3632,11 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, mtCOVERAGE_TEST_MARKER(); } } - kernelEXIT_CRITICAL(); + #if ( portUSING_GRANULAR_LOCKS == 1 ) + vTaskTCBExitCritical(); + #else + kernelEXIT_CRITICAL(); + #endif traceRETURN_vTaskPreemptionDisable(); } @@ -3310,18 +3646,23 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - BaseType_t prvTaskPreemptionEnable( const TaskHandle_t xTask ) + BaseType_t xTaskPreemptionEnableWithYieldStatus( const TaskHandle_t xTask ) { TCB_t * pxTCB; - UBaseType_t uxDeferredAction = 0U; + BaseType_t xHasDeferredAction = pdFALSE; BaseType_t xAlreadyYielded = pdFALSE; + BaseType_t xTaskRequestedToYield = pdFALSE; - traceENTER_vTaskPreemptionEnable( xTask ); - - kernelENTER_CRITICAL(); + #if ( portUSING_GRANULAR_LOCKS == 1 ) + vTaskTCBEnterCritical(); + #else + kernelENTER_CRITICAL(); + #endif { if( xSchedulerRunning != pdFALSE ) { + /* Current task running on the core can not be changed by other core. + * Get TCB from handle is safe to call within TCB critical section. */ pxTCB = prvGetTCBFromHandle( xTask ); configASSERT( pxTCB != NULL ); configASSERT( pxTCB->uxPreemptionDisable > 0U ); @@ -3332,14 +3673,20 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, { if( pxTCB->uxDeferredStateChange != 0U ) { - uxDeferredAction = pxTCB->uxDeferredStateChange; + /* A deferred state change is pending. Record only the fact + * that a deferred action is pending, rather than the specific + * deferred action itself. This allows us to tolerate a concurrent + * vTaskDelete()/vTaskSuspend() on another core that may have + * upgraded the deferred action between the snapshot and the dispatch. + * The actual deferred action will be read and dispatched in the + * loop below after exiting the TCB critical section. */ + xHasDeferredAction = pdTRUE; } else { if( ( xYieldPendings[ pxTCB->xTaskRunState ] != pdFALSE ) && ( taskTASK_IS_RUNNING( pxTCB ) != pdFALSE ) ) { - prvYieldCore( pxTCB->xTaskRunState ); - xAlreadyYielded = pdTRUE; + xTaskRequestedToYield = pdTRUE; } else { @@ -3357,25 +3704,53 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, mtCOVERAGE_TEST_MARKER(); } } - kernelEXIT_CRITICAL(); + #if ( portUSING_GRANULAR_LOCKS == 1 ) + vTaskTCBExitCritical(); + #else + kernelEXIT_CRITICAL(); + #endif - if( uxDeferredAction != 0U ) + if( xHasDeferredAction != pdFALSE ) { - if( uxDeferredAction & tskDEFERRED_DELETION ) + /* Read uxDeferredStateChange after exiting the TCB critical section. + * Between the snapshot and here, another core may have upgraded the + * deferred action. */ + if( pxTCB->uxDeferredStateChange & tskDEFERRED_DELETION ) { vTaskDelete( xTask ); + xAlreadyYielded = pdTRUE; } - else if( uxDeferredAction & tskDEFERRED_SUSPENSION ) + else if( pxTCB->uxDeferredStateChange & tskDEFERRED_SUSPENSION ) { vTaskSuspend( xTask ); + xAlreadyYielded = pdTRUE; } else { + /* Concurrent operation cleared the deferred state — nothing to do. */ mtCOVERAGE_TEST_MARKER(); } + } + else + { + if( xTaskRequestedToYield != pdFALSE ) + { + /* prvYieldCore must be called in critical section. */ + kernelENTER_CRITICAL(); + { + pxTCB = prvGetTCBFromHandle( xTask ); - /* Any deferred action on the task would result in a context switch. */ - xAlreadyYielded = pdTRUE; + /* There is gap between TCB critical section and kernel critical section. + * Checking the yield pending again to prevent that the current task + * already handle the yield request. */ + if( ( xYieldPendings[ pxTCB->xTaskRunState ] != pdFALSE ) && ( taskTASK_IS_RUNNING( pxTCB ) != pdFALSE ) ) + { + prvYieldCore( pxTCB->xTaskRunState ); + } + } + kernelEXIT_CRITICAL(); + xAlreadyYielded = pdTRUE; + } } return xAlreadyYielded; @@ -3389,7 +3764,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, { traceENTER_vTaskPreemptionEnable( xTask ); - ( void ) prvTaskPreemptionEnable( xTask ); + ( void ) xTaskPreemptionEnableWithYieldStatus( xTask ); traceRETURN_vTaskPreemptionEnable(); } @@ -3411,30 +3786,72 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, kernelENTER_CRITICAL(); { + #if ( ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) ) + const BaseType_t xCoreID = portGET_CORE_ID(); + #endif /* ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) */ + /* If null is passed in here then it is the running task that is * being suspended. */ pxTCB = prvGetTCBFromHandle( xTaskToSuspend ); configASSERT( pxTCB != NULL ); - #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + #if ( ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) ) + { + /* Acquire the TCB lock before reading the preemption-disable + * count to avoid a race with the target task running on another + * core (which may be modifying uxPreemptionDisable concurrently). */ + portGET_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); /* If the task has disabled preemption, we need to defer the suspension until the * task enables preemption. The suspension will be performed in vTaskPreemptionEnable(). */ if( pxTCB->uxPreemptionDisable > 0U ) { - pxTCB->uxDeferredStateChange |= tskDEFERRED_SUSPENSION; + /* Only queue the suspension if a deletion is not already deferred. + * Deletion takes priority — if both are requested, the dispatcher + * will only process the deletion. Recording tskDEFERRED_SUSPENSION on top of + * a pending tskDEFERRED_DELETION would have no effect. */ + if( ( pxTCB->uxDeferredStateChange & tskDEFERRED_DELETION ) == 0U ) + { + pxTCB->uxDeferredStateChange |= tskDEFERRED_SUSPENSION; + } + xDeferredSuspension = pdTRUE; + portRELEASE_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); } else { - mtCOVERAGE_TEST_MARKER(); + /* Reset the deferred state change flags */ + pxTCB->uxDeferredStateChange &= ~tskDEFERRED_SUSPENSION; } - #endif /* configUSE_TASK_PREEMPTION_DISABLE */ + } + #endif /* ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) */ #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) if( xDeferredSuspension == pdFALSE ) #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ { + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) && ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) ) + { + /* In SMP + granular-locks builds, an earlier-deferred suspension + * can be dispatched at the same time another core is calling + * vTaskDelete() on the same TCB. The dispatcher in + * xTaskPreemptionEnableWithYieldStatus releases the TCB + * spinlock before calling us, leaving a small window in which + * vTaskDelete can complete its immediate path and move the TCB + * onto xTasksWaitingTermination. If that has happened, the + * deletion takes precedence — abort here rather than moving + * the TCB off the termination list (which would silently drop + * the deletion request and leak the TCB). */ + if( listLIST_ITEM_CONTAINER( &( pxTCB->xStateListItem ) ) == &xTasksWaitingTermination ) + { + portRELEASE_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); + kernelEXIT_CRITICAL(); + traceRETURN_vTaskSuspend(); + return; + } + } + #endif /* ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) && ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + traceTASK_SUSPEND( pxTCB ); /* Remove task from the ready/delayed list and place in the @@ -3515,6 +3932,9 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, } } #endif /* #if ( configNUMBER_OF_CORES > 1 ) */ + #if ( ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) ) + portRELEASE_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); + #endif } } kernelEXIT_CRITICAL(); @@ -3692,7 +4112,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, kernelENTER_CRITICAL(); { #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - + { /* If the task being resumed is in a deferred suspension state, * we simply clear the deferred suspension state and return. */ if( pxTCB->uxDeferredStateChange & tskDEFERRED_SUSPENSION ) @@ -3704,6 +4124,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, { mtCOVERAGE_TEST_MARKER(); } + } #endif /* configUSE_TASK_PREEMPTION_DISABLE */ #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) @@ -3752,6 +4173,10 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, TCB_t * const pxTCB = xTaskToResume; UBaseType_t uxSavedInterruptStatus; + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + BaseType_t xTaskResumed = pdFALSE; + #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + traceENTER_xTaskResumeFromISR( xTaskToResume ); configASSERT( xTaskToResume ); @@ -3779,58 +4204,82 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, /* coverity[misra_c_2012_directive_4_7_violation] */ uxSavedInterruptStatus = kernelENTER_CRITICAL_FROM_ISR(); { - if( prvTaskIsTaskSuspended( pxTCB ) != pdFALSE ) + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) { - traceTASK_RESUME_FROM_ISR( pxTCB ); + /* FIXME : Acquire the TCB lock before reading the preemptions + * disable count. */ - /* Check the ready lists can be accessed. */ - if( uxSchedulerSuspended == ( UBaseType_t ) 0U ) + /* If the task being resumed is in a deferred suspension state, + * we simply clear the deferred suspension state and return. */ + if( pxTCB->uxDeferredStateChange & tskDEFERRED_SUSPENSION ) { - #if ( configNUMBER_OF_CORES == 1 ) - { - /* Ready lists can be accessed so move the task from the - * suspended list to the ready list directly. */ - if( pxTCB->uxPriority > pxCurrentTCB->uxPriority ) - { - xYieldRequired = pdTRUE; - - /* Mark that a yield is pending in case the user is not - * using the return value to initiate a context switch - * from the ISR using the port specific portYIELD_FROM_ISR(). */ - xYieldPendings[ 0 ] = pdTRUE; - } - else - { - mtCOVERAGE_TEST_MARKER(); - } - } - #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ - - ( void ) uxListRemove( &( pxTCB->xStateListItem ) ); - prvAddTaskToReadyList( pxTCB ); + pxTCB->uxDeferredStateChange &= ~tskDEFERRED_SUSPENSION; + xTaskResumed = pdTRUE; } else { - /* The delayed or ready lists cannot be accessed so the task - * is held in the pending ready list until the scheduler is - * unsuspended. */ - vListInsertEnd( &( xPendingReadyList ), &( pxTCB->xEventListItem ) ); + mtCOVERAGE_TEST_MARKER(); } - - #if ( ( configNUMBER_OF_CORES > 1 ) && ( configUSE_PREEMPTION == 1 ) ) - { - prvYieldForTask( pxTCB ); - - if( xYieldPendings[ portGET_CORE_ID() ] != pdFALSE ) - { - xYieldRequired = pdTRUE; - } - } - #endif /* #if ( ( configNUMBER_OF_CORES > 1 ) && ( configUSE_PREEMPTION == 1 ) ) */ } - else + #endif /* configUSE_TASK_PREEMPTION_DISABLE */ + + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + if( xTaskResumed == pdFALSE ) + #endif /* configUSE_TASK_PREEMPTION_DISABLE */ { - mtCOVERAGE_TEST_MARKER(); + if( prvTaskIsTaskSuspended( pxTCB ) != pdFALSE ) + { + traceTASK_RESUME_FROM_ISR( pxTCB ); + + /* Check the ready lists can be accessed. */ + if( uxSchedulerSuspended == ( UBaseType_t ) 0U ) + { + #if ( configNUMBER_OF_CORES == 1 ) + { + /* Ready lists can be accessed so move the task from the + * suspended list to the ready list directly. */ + if( pxTCB->uxPriority > pxCurrentTCB->uxPriority ) + { + xYieldRequired = pdTRUE; + + /* Mark that a yield is pending in case the user is not + * using the return value to initiate a context switch + * from the ISR using the port specific portYIELD_FROM_ISR(). */ + xYieldPendings[ 0 ] = pdTRUE; + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + } + #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ + + ( void ) uxListRemove( &( pxTCB->xStateListItem ) ); + prvAddTaskToReadyList( pxTCB ); + } + else + { + /* The delayed or ready lists cannot be accessed so the task + * is held in the pending ready list until the scheduler is + * unsuspended. */ + vListInsertEnd( &( xPendingReadyList ), &( pxTCB->xEventListItem ) ); + } + + #if ( ( configNUMBER_OF_CORES > 1 ) && ( configUSE_PREEMPTION == 1 ) ) + { + prvYieldForTask( pxTCB ); + + if( xYieldPendings[ portGET_CORE_ID() ] != pdFALSE ) + { + xYieldRequired = pdTRUE; + } + } + #endif /* #if ( ( configNUMBER_OF_CORES > 1 ) && ( configUSE_PREEMPTION == 1 ) ) */ + } + else + { + mtCOVERAGE_TEST_MARKER(); + } } } kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); @@ -4217,6 +4666,13 @@ void vTaskSuspendAll( void ) kernelGET_TASK_LOCK( xCoreID ); + /* Task lock is acquired here to increase the scheduler suspended count. + * In granular lock, it is possible that prvYieldCore is called with only + * ISR spinlock is acquired in ISR. To remove the gap, the logic is updated + * to acquire the ISR spinlock before check for run state. The logic in + * prvCheckForRunStateChange() is updated accordingly. */ + kernelGET_ISR_LOCK( xCoreID ); + /* uxSchedulerSuspended is increased after prvCheckForRunStateChange. The * purpose is to prevent altering the variable when fromISR APIs are readying * it. */ @@ -4225,9 +4681,14 @@ void vTaskSuspendAll( void ) /* If the scheduler is being suspended after taking a granular lock (in any data group) * then we do not check for the run state of a task and we let it run through until * the granular lock is released. */ + + /* Task may acquire kernel lock in data group critical section. + * In this case, check for run state change is not required since + * we can't yield for other task when preemption is disabled. */ #if ( portUSING_GRANULAR_LOCKS == 1 ) && ( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + && ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) ) { prvCheckForRunStateChange(); @@ -4242,8 +4703,6 @@ void vTaskSuspendAll( void ) * task lock for the core is acquired in prvCheckForRunStateChange. */ xCoreID = ( BaseType_t ) portGET_CORE_ID(); - kernelGET_ISR_LOCK( xCoreID ); - /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment * is used to allow calls to vTaskSuspendAll() to nest. */ ++uxSchedulerSuspended; @@ -4388,9 +4847,9 @@ BaseType_t xTaskResumeAll( void ) } #else /* #if ( configNUMBER_OF_CORES == 1 ) */ { - /* All appropriate tasks yield at the moment a task is added to xPendingReadyList. - * If the current core yielded then vTaskSwitchContext() has already been called - * which sets xYieldPendings for the current core to pdTRUE. */ + /* Yield for the task when removing out from pending + * ready list. */ + prvYieldForTask( pxTCB ); } #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ } @@ -5053,6 +5512,10 @@ BaseType_t xTaskIncrementTick( void ) UBaseType_t uxSavedInterruptStatus; #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configUSE_TICK_HOOK == 1 ) ) + BaseType_t xApplicationTickRequired = pdFALSE; + #endif + traceENTER_xTaskIncrementTick(); #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) @@ -5227,7 +5690,15 @@ BaseType_t xTaskIncrementTick( void ) * count is being unwound (when the scheduler is being unlocked). */ if( xPendedTicks == ( TickType_t ) 0 ) { - vApplicationTickHook(); + #if ( portUSING_GRANULAR_LOCKS == 1 ) + { + xApplicationTickRequired = pdTRUE; + } + #else + { + vApplicationTickHook(); + } + #endif } else { @@ -5291,15 +5762,34 @@ BaseType_t xTaskIncrementTick( void ) * scheduler is locked. */ #if ( configUSE_TICK_HOOK == 1 ) { - vApplicationTickHook(); + #if ( portUSING_GRANULAR_LOCKS == 1 ) + { + xApplicationTickRequired = pdTRUE; + } + #else + { + vApplicationTickHook(); + } + #endif } - #endif + #endif /* configUSE_TICK_HOOK */ } #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configUSE_TICK_HOOK == 1 ) ) + { + /* In order not to voilate granular lock critical section hierarchy, calling + * the vApplicationTickHook when leaving the kernel critical section. */ + if( xApplicationTickRequired == pdTRUE ) + { + vApplicationTickHook(); + } + } + #endif + traceRETURN_xTaskIncrementTick( xSwitchRequired ); return xSwitchRequired; @@ -5545,13 +6035,11 @@ BaseType_t xTaskIncrementTick( void ) * SMP port. */ configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 ); - /* vTaskSwitchContext() must not be called with a task that has - * preemption disabled. */ - #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - configASSERT( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ); - #endif - - if( uxSchedulerSuspended != ( UBaseType_t ) 0U ) + if( ( uxSchedulerSuspended != ( UBaseType_t ) 0U ) + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + || ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable != 0U ) + #endif + ) { /* The scheduler is currently suspended or the task * has requested to not be preempted - do not allow @@ -5769,137 +6257,152 @@ void vTaskPlaceOnUnorderedEventList( List_t * pxEventList, BaseType_t xTaskRemoveFromEventList( const List_t * const pxEventList ) { - TCB_t * pxUnblockedTCB; BaseType_t xReturn; - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - UBaseType_t uxSavedInterruptStatus; - #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ - traceENTER_xTaskRemoveFromEventList( pxEventList ); - #if ( !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) ) - - /* THIS FUNCTION MUST BE CALLED FROM A CRITICAL SECTION. It can also be - * called from a critical section within an ISR. */ - #else /* #if ( ! ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) ) */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) /* Lock the kernel data group as we are about to access its members */ - if( portCHECK_IF_IN_ISR() == pdTRUE ) + prvKernelEnterISROnlyCritical(); { - uxSavedInterruptStatus = kernelENTER_CRITICAL_FROM_ISR(); + xReturn = prvTaskRemoveFromEventList( pxEventList ); + } + prvKernelExitISROnlyCritical(); + #else + xReturn = prvTaskRemoveFromEventList( pxEventList ); + #endif + + traceRETURN_xTaskRemoveFromEventList( xReturn ); + return xReturn; +} + +/*-----------------------------------------------------------*/ + +BaseType_t xTaskRemoveFromEventListFromISR( const List_t * const pxEventList ) +{ + BaseType_t xReturn; + + traceENTER_xTaskRemoveFromEventListFromISR( pxEventList ); + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + { + UBaseType_t uxSavedInterruptStatus; + + /* Lock the kernel data group as we are about to access its members */ + uxSavedInterruptStatus = kernelENTER_CRITICAL_FROM_ISR(); + { + xReturn = prvTaskRemoveFromEventList( pxEventList ); + } + kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + } + #else /* if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + { + xReturn = prvTaskRemoveFromEventList( pxEventList ); + } + #endif /* if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + + traceRETURN_xTaskRemoveFromEventListFromISR( xReturn ); + return xReturn; +} + +/*-----------------------------------------------------------*/ + +static BaseType_t prvTaskRemoveFromEventList( const List_t * const pxEventList ) +{ + TCB_t * pxUnblockedTCB; + BaseType_t xReturn; + + /* THIS FUNCTION MUST BE CALLED FROM A CRITICAL SECTION. It can also be + * called from a critical section within an ISR. */ + + /* Before proceeding, check if the event list is empty */ + if( listLIST_IS_EMPTY( pxEventList ) == pdFALSE ) + { + /* The event list is sorted in priority order, so the first in the list can + * be removed as it is known to be the highest priority. Remove the TCB from + * the delayed list, and add it to the ready list. + * + * If an event is for a queue that is locked then this function will never + * get called - the lock count on the queue will get modified instead. This + * means exclusive access to the event list is guaranteed here. + * + * This function assumes that a check has already been made to ensure that + * pxEventList is not empty. */ + /* MISRA Ref 11.5.3 [Void pointer assignment] */ + /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-115 */ + /* coverity[misra_c_2012_rule_11_5_violation] */ + pxUnblockedTCB = listGET_OWNER_OF_HEAD_ENTRY( pxEventList ); + configASSERT( pxUnblockedTCB ); + listREMOVE_ITEM( &( pxUnblockedTCB->xEventListItem ) ); + + if( uxSchedulerSuspended == ( UBaseType_t ) 0U ) + { + listREMOVE_ITEM( &( pxUnblockedTCB->xStateListItem ) ); + prvAddTaskToReadyList( pxUnblockedTCB ); + + #if ( configUSE_TICKLESS_IDLE != 0 ) + { + /* If a task is blocked on a kernel object then xNextTaskUnblockTime + * might be set to the blocked task's time out time. If the task is + * unblocked for a reason other than a timeout xNextTaskUnblockTime is + * normally left unchanged, because it is automatically reset to a new + * value when the tick count equals xNextTaskUnblockTime. However if + * tickless idling is used it might be more important to enter sleep mode + * at the earliest possible time - so reset xNextTaskUnblockTime here to + * ensure it is updated at the earliest possible time. */ + prvResetNextTaskUnblockTime(); + } + #endif } else { - uxSavedInterruptStatus = 0; - kernelENTER_CRITICAL(); + /* The delayed and ready lists cannot be accessed, so hold this task + * pending until the scheduler is resumed. */ + listINSERT_END( &( xPendingReadyList ), &( pxUnblockedTCB->xEventListItem ) ); } - /* Before taking the kernel lock, another task/ISR could have already - * emptied the pxEventList. So we insert a check here to see if - * pxEventList is empty before attempting to remove an item from it. */ - if( listLIST_IS_EMPTY( pxEventList ) == pdFALSE ) + #if ( configNUMBER_OF_CORES == 1 ) { - #endif /* #if ( ! ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) ) */ + if( pxUnblockedTCB->uxPriority > pxCurrentTCB->uxPriority ) + { + /* Return true if the task removed from the event list has a higher + * priority than the calling task. This allows the calling task to know if + * it should force a context switch now. */ + xReturn = pdTRUE; - /* The event list is sorted in priority order, so the first in the list can - * be removed as it is known to be the highest priority. Remove the TCB from - * the delayed list, and add it to the ready list. - * - * If an event is for a queue that is locked then this function will never - * get called - the lock count on the queue will get modified instead. This - * means exclusive access to the event list is guaranteed here. - * - * This function assumes that a check has already been made to ensure that - * pxEventList is not empty. */ - /* MISRA Ref 11.5.3 [Void pointer assignment] */ - /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-115 */ - /* coverity[misra_c_2012_rule_11_5_violation] */ - pxUnblockedTCB = listGET_OWNER_OF_HEAD_ENTRY( pxEventList ); - configASSERT( pxUnblockedTCB ); - listREMOVE_ITEM( &( pxUnblockedTCB->xEventListItem ) ); - - if( uxSchedulerSuspended == ( UBaseType_t ) 0U ) - { - listREMOVE_ITEM( &( pxUnblockedTCB->xStateListItem ) ); - prvAddTaskToReadyList( pxUnblockedTCB ); - - #if ( configUSE_TICKLESS_IDLE != 0 ) - { - /* If a task is blocked on a kernel object then xNextTaskUnblockTime - * might be set to the blocked task's time out time. If the task is - * unblocked for a reason other than a timeout xNextTaskUnblockTime is - * normally left unchanged, because it is automatically reset to a new - * value when the tick count equals xNextTaskUnblockTime. However if - * tickless idling is used it might be more important to enter sleep mode - * at the earliest possible time - so reset xNextTaskUnblockTime here to - * ensure it is updated at the earliest possible time. */ - prvResetNextTaskUnblockTime(); + /* Mark that a yield is pending in case the user is not using the + * "xHigherPriorityTaskWoken" parameter to an ISR safe FreeRTOS function. */ + xYieldPendings[ 0 ] = pdTRUE; + } + else + { + xReturn = pdFALSE; + } } - #endif + #else /* #if ( configNUMBER_OF_CORES == 1 ) */ + { + xReturn = pdFALSE; + + #if ( configUSE_PREEMPTION == 1 ) + { + prvYieldForTask( pxUnblockedTCB ); + + if( xYieldPendings[ portGET_CORE_ID() ] != pdFALSE ) + { + xReturn = pdTRUE; + } + } + #endif /* #if ( configUSE_PREEMPTION == 1 ) */ + } + #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ } else { - /* The delayed and ready lists cannot be accessed, so hold this task - * pending until the scheduler is resumed. */ - listINSERT_END( &( xPendingReadyList ), &( pxUnblockedTCB->xEventListItem ) ); - } - - #if ( configNUMBER_OF_CORES == 1 ) - { - if( pxUnblockedTCB->uxPriority > pxCurrentTCB->uxPriority ) - { - /* Return true if the task removed from the event list has a higher - * priority than the calling task. This allows the calling task to know if - * it should force a context switch now. */ - xReturn = pdTRUE; - - /* Mark that a yield is pending in case the user is not using the - * "xHigherPriorityTaskWoken" parameter to an ISR safe FreeRTOS function. */ - xYieldPendings[ 0 ] = pdTRUE; - } - else - { - xReturn = pdFALSE; - } - } - #else /* #if ( configNUMBER_OF_CORES == 1 ) */ - { + /* The pxEventList was emptied before we entered the critical + * section, Nothing to do except return pdFALSE. */ xReturn = pdFALSE; - - #if ( configUSE_PREEMPTION == 1 ) - { - prvYieldForTask( pxUnblockedTCB ); - - if( xYieldPendings[ portGET_CORE_ID() ] != pdFALSE ) - { - xReturn = pdTRUE; - } - } - #endif /* #if ( configUSE_PREEMPTION == 1 ) */ } - #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) -} -else -{ - /* The pxEventList was emptied before we entered the critical - * section, Nothing to do except return pdFALSE. */ - xReturn = pdFALSE; -} - -/* We are done accessing the kernel data group. Unlock it. */ -if( portCHECK_IF_IN_ISR() == pdTRUE ) -{ - kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); -} -else -{ - kernelEXIT_CRITICAL(); -} - #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ - - traceRETURN_xTaskRemoveFromEventList( xReturn ); return xReturn; } /*-----------------------------------------------------------*/ @@ -5911,9 +6414,12 @@ void vTaskRemoveFromUnorderedEventList( ListItem_t * pxEventListItem, traceENTER_vTaskRemoveFromUnorderedEventList( pxEventListItem, xItemValue ); - /* THIS FUNCTION MUST BE CALLED WITH THE SCHEDULER SUSPENDED. It is used by - * the event flags implementation. */ - configASSERT( uxSchedulerSuspended != ( UBaseType_t ) 0U ); + #if ( !( portUSING_GRANULAR_LOCKS == 1 ) ) + + /* THIS FUNCTION MUST BE CALLED WITH THE SCHEDULER SUSPENDED. It is used by + * the event flags implementation. */ + configASSERT( uxSchedulerSuspended != ( UBaseType_t ) 0U ); + #endif /* #if ( ! ( portUSING_GRANULAR_LOCKS == 1 ) ) */ /* Store the new item value in the event list. */ listSET_LIST_ITEM_VALUE( pxEventListItem, xItemValue | taskEVENT_LIST_ITEM_VALUE_IN_USE ); @@ -5998,7 +6504,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) /* Lock the kernel data group as we are about to access its members */ - kernelENTER_CRITICAL(); + prvKernelEnterISROnlyCritical(); #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /* For internal use only as it does not use a critical section. */ @@ -6007,7 +6513,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) /* We are done accessing the kernel data group. Unlock it. */ - kernelEXIT_CRITICAL(); + prvKernelExitISROnlyCritical(); #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ traceRETURN_vTaskInternalSetTimeOutState(); @@ -7431,7 +7937,16 @@ STATIC void prvResetNextTaskUnblockTime( void ) { const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); - if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + if( ( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + + /* Task yield can be called in a data group critilcal section. + * Adding a preemption disable check to prevent invalid context + * switch. */ + && ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) && + ( pxCurrentTCBs[ xCoreID ]->uxDeferredStateChange == 0U ) + #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + ) { portYIELD(); } @@ -7521,7 +8036,8 @@ STATIC void prvResetNextTaskUnblockTime( void ) * interrupt. Only assert if the critical nesting count is 1 to * protect against recursive calls if the assert function also uses a * critical section. */ - if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 1U ) + if( ( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 1U ) && + ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) ) { portASSERT_IF_IN_ISR(); @@ -7658,7 +8174,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) BaseType_t xYieldCurrentTask; /* Get the xYieldPending status inside the critical section. */ - if( ( xYieldPendings[ xCoreID ] == pdTRUE ) && ( uxSchedulerSuspended == pdFALSE ) + if( ( xYieldPendings[ xCoreID ] == pdTRUE ) #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) && ( pxCurrentTCBs[ xCoreID ]->uxDeferredStateChange == 0U ) @@ -7800,6 +8316,41 @@ STATIC void prvResetNextTaskUnblockTime( void ) #endif /* #if ( configNUMBER_OF_CORES > 1 ) */ /*-----------------------------------------------------------*/ +/* ISR only critical can only be used when multi-critical section is used. + * Therefore, run state change is not valid due to task can't be requested to yield. */ +static void prvKernelEnterISROnlyCritical( void ) +{ + if( xSchedulerRunning != pdFALSE ) + { + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + + configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ); + + /* Take only the ISR lock, not the task lock. */ + kernelGET_ISR_LOCK( xCoreID ); + + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + } +} +/*-----------------------------------------------------------*/ + +static void prvKernelExitISROnlyCritical( void ) +{ + if( xSchedulerRunning != pdFALSE ) + { + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + + configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ); + + /* Release only the ISR lock. */ + kernelRELEASE_ISR_LOCK( xCoreID ); + + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + } +} + +/*-----------------------------------------------------------*/ + #if ( configUSE_STATS_FORMATTING_FUNCTIONS > 0 ) STATIC char * prvWriteNameToBuffer( char * pcBuffer, @@ -8730,6 +9281,9 @@ TickType_t uxTaskResetEventItemValue( void ) #else /* #if ( configNUMBER_OF_CORES == 1 ) */ { #if ( configUSE_PREEMPTION == 1 ) + + /* Yield for the unblocked task is required even when scheduler + * is suspended. */ { prvYieldForTask( pxTCB ); @@ -8864,6 +9418,9 @@ TickType_t uxTaskResetEventItemValue( void ) #else /* #if ( configNUMBER_OF_CORES == 1 ) */ { #if ( configUSE_PREEMPTION == 1 ) + + /* Yield for the unblocked task is required even when scheduler + * is suspended. */ { prvYieldForTask( pxTCB ); @@ -9411,3 +9968,32 @@ void vTaskResetState( void ) #endif /* #if ( configGENERATE_RUN_TIME_STATS == 1 ) */ } /*-----------------------------------------------------------*/ + +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + + BaseType_t xTaskUnlockCanYield( void ) + { + BaseType_t xReturn; + BaseType_t xCoreID = portGET_CORE_ID(); + + /* Current core to call this function should handle the yield request when + * the scheduler is suspended. In vTaskSwitchContext, the core will be blocking + * on TASK spinlocks until scheduler is resumed. */ + if( ( xYieldPendings[ xCoreID ] == pdTRUE ) + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + && ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) + #endif /* ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + ) + { + xReturn = pdTRUE; + } + else + { + xReturn = pdFALSE; + } + + return xReturn; + } + +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ +/*-----------------------------------------------------------*/ diff --git a/timers.c b/timers.c index 1bc40bc46..b283522cb 100644 --- a/timers.c +++ b/timers.c @@ -149,6 +149,16 @@ PRIVILEGED_DATA static QueueHandle_t xTimerQueue = NULL; PRIVILEGED_DATA static TaskHandle_t xTimerTaskHandle = NULL; + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #ifdef portREMOVE_STATIC_QUALIFIER + PRIVILEGED_DATA portSPINLOCK_TYPE xTimerTaskSpinlock = portINIT_SPINLOCK_STATIC; + PRIVILEGED_DATA portSPINLOCK_TYPE xTimerISRSpinlock = portINIT_SPINLOCK_STATIC; + #else + PRIVILEGED_DATA static portSPINLOCK_TYPE xTimerTaskSpinlock = portINIT_SPINLOCK_STATIC; + PRIVILEGED_DATA static portSPINLOCK_TYPE xTimerISRSpinlock = portINIT_SPINLOCK_STATIC; + #endif + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + /*-----------------------------------------------------------*/ /* From 813017082e268c76ab3ae7e136169e8ed89445f7 Mon Sep 17 00:00:00 2001 From: Darian Leung Date: Sun, 16 Jun 2024 00:53:03 +0800 Subject: [PATCH 07/11] change(freertos/smp): Update queue.c locking Updated queue.c to use granular locking - Added xTaskSpinlock and xISRSpinlock - Replaced critical section macros with data group critical section macros such as taskENTER/EXIT_CRITICAL/_FROM_ISR() with queueENTER/EXIT_CRITICAL_FROM_ISR(). - Added vQueueEnterCritical/FromISR() and vQueueExitCritical/FromISR() which map to the data group critical section macros. - Added prvLockQueueForTasks() and prvUnlockQueueForTasks() as the granular locking equivalents to prvLockQueue() and prvUnlockQueue() respectively Co-authored-by: Sudeep Mohanty --- include/FreeRTOS.h | 19 ++ queue.c | 513 ++++++++++++++++++++++++--------------------- 2 files changed, 288 insertions(+), 244 deletions(-) diff --git a/include/FreeRTOS.h b/include/FreeRTOS.h index dc2748d97..d9a3542cc 100644 --- a/include/FreeRTOS.h +++ b/include/FreeRTOS.h @@ -2150,10 +2150,18 @@ #define traceENTER_xTaskRemoveFromEventList( pxEventList ) #endif +#ifndef traceENTER_xTaskRemoveFromEventListFromISR + #define traceENTER_xTaskRemoveFromEventListFromISR( pxEventList ) +#endif + #ifndef traceRETURN_xTaskRemoveFromEventList #define traceRETURN_xTaskRemoveFromEventList( xReturn ) #endif +#ifndef traceRETURN_xTaskRemoveFromEventListFromISR + #define traceRETURN_xTaskRemoveFromEventListFromISR( xReturn ) +#endif + #ifndef traceENTER_vTaskRemoveFromUnorderedEventList #define traceENTER_vTaskRemoveFromUnorderedEventList( pxEventListItem, xItemValue ) #endif @@ -3324,6 +3332,10 @@ typedef struct xSTATIC_QUEUE UBaseType_t uxDummy8; uint8_t ucDummy9; #endif + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + portSPINLOCK_TYPE xDummySpinlock[ 2 ]; + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ } StaticQueue_t; typedef StaticQueue_t StaticSemaphore_t; @@ -3353,6 +3365,10 @@ typedef struct xSTATIC_EVENT_GROUP #if ( ( configSUPPORT_STATIC_ALLOCATION == 1 ) && ( configSUPPORT_DYNAMIC_ALLOCATION == 1 ) ) uint8_t ucDummy4; #endif + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + portSPINLOCK_TYPE xDummySpinlock[ 2 ]; + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ } StaticEventGroup_t; /* @@ -3408,6 +3424,9 @@ typedef struct xSTATIC_STREAM_BUFFER void * pvDummy5[ 2 ]; #endif UBaseType_t uxDummy6; + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + portSPINLOCK_TYPE xDummySpinlock[ 2 ]; + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ } StaticStreamBuffer_t; /* Message buffers are built on stream buffers. */ diff --git a/queue.c b/queue.c index f8198348d..69277851d 100644 --- a/queue.c +++ b/queue.c @@ -133,6 +133,11 @@ typedef struct QueueDefinition /* The old naming convention is used to prevent b UBaseType_t uxQueueNumber; uint8_t ucQueueType; #endif + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + portSPINLOCK_TYPE xTaskSpinlock; + portSPINLOCK_TYPE xISRSpinlock; + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ } xQUEUE; /* The old xQUEUE name is maintained above then typedefed to the new Queue_t @@ -187,7 +192,7 @@ static void prvUnlockQueue( Queue_t * const pxQueue ) PRIVILEGED_FUNCTION; * * @return pdTRUE if the queue contains no items, otherwise pdFALSE. */ -static BaseType_t prvIsQueueEmpty( const Queue_t * pxQueue ) PRIVILEGED_FUNCTION; +static BaseType_t prvIsQueueEmpty( Queue_t * pxQueue ) PRIVILEGED_FUNCTION; /* * Uses a critical section to determine if there is any space in a queue. @@ -217,7 +222,20 @@ static void prvCopyDataFromQueue( Queue_t * const pxQueue, * the queue set that the queue contains data. */ static BaseType_t prvNotifyQueueSetContainer( const Queue_t * const pxQueue ) PRIVILEGED_FUNCTION; -#endif + +/* + * A version of prvNotifyQueueSetContainer() that can be called from an + * interrupt service routine (ISR). + */ + static BaseType_t prvNotifyQueueSetContainerFromISR( const Queue_t * const pxQueue ) PRIVILEGED_FUNCTION; + +/* + * This function serves as a generic implementation for prvNotifyQueueSetContainer() + * and prvNotifyQueueSetContainerFromISR(). + */ + static BaseType_t prvNotifyQueueSetContainerGeneric( const Queue_t * const pxQueue, + const BaseType_t xIsISR ) PRIVILEGED_FUNCTION; +#endif /* if ( configUSE_QUEUE_SETS == 1 ) */ /* * Called after a Queue_t structure has been allocated either statically or @@ -251,12 +269,59 @@ static void prvInitialiseNewQueue( const UBaseType_t uxQueueLength, #endif /*-----------------------------------------------------------*/ +/* + * Macros to mark the start and end of a critical code region. + */ +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define queueENTER_CRITICAL( pxQueue ) taskDATA_GROUP_ENTER_CRITICAL( ( portSPINLOCK_TYPE * ) &( pxQueue )->xTaskSpinlock, ( portSPINLOCK_TYPE * ) &( pxQueue )->xISRSpinlock ) + #define queueENTER_CRITICAL_FROM_ISR( pxQueue, puxSavedInterruptStatus ) taskDATA_GROUP_ENTER_CRITICAL_FROM_ISR( ( portSPINLOCK_TYPE * ) &( pxQueue )->xISRSpinlock, puxSavedInterruptStatus ) + #define queueEXIT_CRITICAL( pxQueue ) taskDATA_GROUP_EXIT_CRITICAL( ( portSPINLOCK_TYPE * ) &( pxQueue )->xTaskSpinlock, ( portSPINLOCK_TYPE * ) &( pxQueue )->xISRSpinlock ) + #define queueEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxQueue ) taskDATA_GROUP_EXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, ( portSPINLOCK_TYPE * ) &( pxQueue )->xISRSpinlock ) +#else /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + #define queueENTER_CRITICAL( pxQueue ) taskENTER_CRITICAL() + #define queueENTER_CRITICAL_FROM_ISR( pxQueue, puxSavedInterruptStatus ) do { *( puxSavedInterruptStatus ) = taskENTER_CRITICAL_FROM_ISR(); } while( 0 ) + #define queueEXIT_CRITICAL( pxQueue ) taskEXIT_CRITICAL() + #define queueEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxQueue ) taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + /* * Macro to mark a queue as locked. Locking a queue prevents an ISR from * accessing the queue event lists. + * + * Under granular locks, queueLOCK()/queueUNLOCK() already surround this with + * task-level spinlock + preemption-disabled region. To avoid redundant + * data-group enter/exit (which would re-disable preemption and re-acquire the + * same task spinlock), we minimize to interrupt masking only for the small + * metadata updates here. For the non-granular path, retain the full + * queueENTER_CRITICAL/queueEXIT_CRITICAL. */ -#define prvLockQueue( pxQueue ) \ - taskENTER_CRITICAL(); \ +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #define prvLockQueue( pxQueue ) \ + do { \ + UBaseType_t ulState = portSET_INTERRUPT_MASK(); \ + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( ( pxQueue )->xISRSpinlock ) ); \ + { \ + if( ( pxQueue )->cRxLock == queueUNLOCKED ) \ + { \ + ( pxQueue )->cRxLock = queueLOCKED_UNMODIFIED; \ + } \ + if( ( pxQueue )->cTxLock == queueUNLOCKED ) \ + { \ + ( pxQueue )->cTxLock = queueLOCKED_UNMODIFIED; \ + } \ + } \ + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( ( pxQueue )->xISRSpinlock ) ); \ + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 ) \ + { \ + portCLEAR_INTERRUPT_MASK( ulState ); \ + } \ + } while( 0 ) +#else /* if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #define prvLockQueue( pxQueue ) \ + queueENTER_CRITICAL( pxQueue ); \ { \ if( ( pxQueue )->cRxLock == queueUNLOCKED ) \ { \ @@ -267,7 +332,8 @@ static void prvInitialiseNewQueue( const UBaseType_t uxQueueLength, ( pxQueue )->cTxLock = queueLOCKED_UNMODIFIED; \ } \ } \ - taskEXIT_CRITICAL() + queueEXIT_CRITICAL( pxQueue ) +#endif /* if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /* * Macro to increment cTxLock member of the queue data structure. It is @@ -298,6 +364,56 @@ static void prvInitialiseNewQueue( const UBaseType_t uxQueueLength, ( pxQueue )->cRxLock = ( int8_t ) ( ( cRxLock ) + ( int8_t ) 1 ); \ } \ } while( 0 ) + +/* + * Macro used to lock and unlock a queue. When a task locks a queue, the + * task will have thread safe non-deterministic access to the queue. + * - Concurrent access from other tasks will be blocked by the xTaskSpinlock + * - Concurrent access from ISRs will be pended + * + * When the tasks unlocks the queue, all pended access attempts are handled. + */ +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #define queueLOCK( pxQueue ) \ + do { \ + taskDATA_GROUP_LOCK( &( ( pxQueue )->xTaskSpinlock ) ); \ + prvLockQueue( ( pxQueue ) ); \ + } while( 0 ) + #define queueUNLOCK( pxQueue, xYieldAPI ) \ + do { \ + BaseType_t xAlreadyYielded; \ + prvUnlockQueue( ( pxQueue ) ); \ + xAlreadyYielded = taskDATA_GROUP_UNLOCK( &( ( pxQueue )->xTaskSpinlock ) ); \ + if( ( xAlreadyYielded == pdFALSE ) && ( ( xYieldAPI ) == pdTRUE ) ) \ + { \ + taskYIELD_WITHIN_API(); \ + } \ + else \ + { \ + mtCOVERAGE_TEST_MARKER(); \ + } \ + } while( 0 ) +#else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #define queueLOCK( pxQueue ) \ + do { \ + vTaskSuspendAll(); \ + prvLockQueue( ( pxQueue ) ); \ + } while( 0 ) + #define queueUNLOCK( pxQueue, xYieldAPI ) \ + do { \ + BaseType_t xAlreadyYielded; \ + prvUnlockQueue( ( pxQueue ) ); \ + xAlreadyYielded = xTaskResumeAll(); \ + if( ( xAlreadyYielded == pdFALSE ) && ( ( xYieldAPI ) == pdTRUE ) ) \ + { \ + taskYIELD_WITHIN_API(); \ + } \ + else \ + { \ + mtCOVERAGE_TEST_MARKER(); \ + } \ + } while( 0 ) +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /*-----------------------------------------------------------*/ BaseType_t xQueueGenericReset( QueueHandle_t xQueue, @@ -310,12 +426,22 @@ BaseType_t xQueueGenericReset( QueueHandle_t xQueue, configASSERT( pxQueue ); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + { + if( xNewQueue == pdTRUE ) + { + portINIT_SPINLOCK( &( pxQueue->xTaskSpinlock ) ); + portINIT_SPINLOCK( &( pxQueue->xISRSpinlock ) ); + } + } + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + if( ( pxQueue != NULL ) && ( pxQueue->uxLength >= 1U ) && /* Check for multiplication overflow. */ ( ( SIZE_MAX / pxQueue->uxLength ) >= pxQueue->uxItemSize ) ) { - taskENTER_CRITICAL(); + queueENTER_CRITICAL( pxQueue ); { pxQueue->u.xQueue.pcTail = pxQueue->pcHead + ( pxQueue->uxLength * pxQueue->uxItemSize ); pxQueue->uxMessagesWaiting = ( UBaseType_t ) 0U; @@ -331,16 +457,9 @@ BaseType_t xQueueGenericReset( QueueHandle_t xQueue, * will still be empty. If there are tasks blocked waiting to write to * the queue, then one should be unblocked as after this function exits * it will be possible to write to it. */ - if( listLIST_IS_EMPTY( &( pxQueue->xTasksWaitingToSend ) ) == pdFALSE ) + if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToSend ) ) != pdFALSE ) { - if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToSend ) ) != pdFALSE ) - { - queueYIELD_IF_USING_PREEMPTION(); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + queueYIELD_IF_USING_PREEMPTION(); } else { @@ -354,7 +473,7 @@ BaseType_t xQueueGenericReset( QueueHandle_t xQueue, vListInitialise( &( pxQueue->xTasksWaitingToReceive ) ); } } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); } else { @@ -703,7 +822,7 @@ static void prvInitialiseNewQueue( const UBaseType_t uxQueueLength, * calling task is the mutex holder, but not a good way of determining the * identity of the mutex holder, as the holder may change between the * following critical section exiting and the function returning. */ - taskENTER_CRITICAL(); + queueENTER_CRITICAL( pxSemaphore ); { if( pxSemaphore->uxQueueType == queueQUEUE_IS_MUTEX ) { @@ -714,7 +833,7 @@ static void prvInitialiseNewQueue( const UBaseType_t uxQueueLength, pxReturn = NULL; } } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxSemaphore ); traceRETURN_xQueueGetMutexHolder( pxReturn ); @@ -968,7 +1087,7 @@ BaseType_t xQueueGenericSend( QueueHandle_t xQueue, for( ; ; ) { - taskENTER_CRITICAL(); + queueENTER_CRITICAL( pxQueue ); { /* Is there room on the queue now? The running task must be the * highest priority task wanting to access the queue. If the head item @@ -1009,20 +1128,13 @@ BaseType_t xQueueGenericSend( QueueHandle_t xQueue, { /* If there was a task waiting for data to arrive on the * queue then unblock it now. */ - if( listLIST_IS_EMPTY( &( pxQueue->xTasksWaitingToReceive ) ) == pdFALSE ) + if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) { - if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) - { - /* The unblocked task has a priority higher than - * our own so yield immediately. Yes it is ok to - * do this from within the critical section - the - * kernel takes care of that. */ - queueYIELD_IF_USING_PREEMPTION(); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + /* The unblocked task has a priority higher than + * our own so yield immediately. Yes it is ok to + * do this from within the critical section - the + * kernel takes care of that. */ + queueYIELD_IF_USING_PREEMPTION(); } else if( xYieldRequired != pdFALSE ) { @@ -1044,20 +1156,13 @@ BaseType_t xQueueGenericSend( QueueHandle_t xQueue, /* If there was a task waiting for data to arrive on the * queue then unblock it now. */ - if( listLIST_IS_EMPTY( &( pxQueue->xTasksWaitingToReceive ) ) == pdFALSE ) + if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) { - if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) - { - /* The unblocked task has a priority higher than - * our own so yield immediately. Yes it is ok to do - * this from within the critical section - the kernel - * takes care of that. */ - queueYIELD_IF_USING_PREEMPTION(); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + /* The unblocked task has a priority higher than + * our own so yield immediately. Yes it is ok to do + * this from within the critical section - the kernel + * takes care of that. */ + queueYIELD_IF_USING_PREEMPTION(); } else if( xYieldRequired != pdFALSE ) { @@ -1074,7 +1179,7 @@ BaseType_t xQueueGenericSend( QueueHandle_t xQueue, } #endif /* configUSE_QUEUE_SETS */ - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); traceRETURN_xQueueGenericSend( pdPASS ); @@ -1086,7 +1191,7 @@ BaseType_t xQueueGenericSend( QueueHandle_t xQueue, { /* The queue was full and no block time is specified (or * the block time has expired) so leave now. */ - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); /* Return to the original privilege level before exiting * the function. */ @@ -1109,13 +1214,12 @@ BaseType_t xQueueGenericSend( QueueHandle_t xQueue, } } } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); /* Interrupts and other tasks can send to and receive from the queue * now the critical section has been exited. */ - vTaskSuspendAll(); - prvLockQueue( pxQueue ); + queueLOCK( pxQueue ); /* Update the timeout state to see if it has expired yet. */ if( xTaskCheckForTimeOut( &xTimeOut, &xTicksToWait ) == pdFALSE ) @@ -1125,35 +1229,18 @@ BaseType_t xQueueGenericSend( QueueHandle_t xQueue, traceBLOCKING_ON_QUEUE_SEND( pxQueue ); vTaskPlaceOnEventList( &( pxQueue->xTasksWaitingToSend ), xTicksToWait ); - /* Unlocking the queue means queue events can effect the - * event list. It is possible that interrupts occurring now - * remove this task from the event list again - but as the - * scheduler is suspended the task will go onto the pending - * ready list instead of the actual ready list. */ - prvUnlockQueue( pxQueue ); - - /* Resuming the scheduler will move tasks from the pending - * ready list into the ready list - so it is feasible that this - * task is already in the ready list before it yields - in which - * case the yield will not cause a context switch unless there - * is also a higher priority task in the pending ready list. */ - if( xTaskResumeAll() == pdFALSE ) - { - taskYIELD_WITHIN_API(); - } + queueUNLOCK( pxQueue, pdTRUE ); } else { /* Try again. */ - prvUnlockQueue( pxQueue ); - ( void ) xTaskResumeAll(); + queueUNLOCK( pxQueue, pdFALSE ); } } else { /* The timeout has expired. */ - prvUnlockQueue( pxQueue ); - ( void ) xTaskResumeAll(); + queueUNLOCK( pxQueue, pdFALSE ); traceQUEUE_SEND_FAILED( pxQueue ); traceRETURN_xQueueGenericSend( errQUEUE_FULL ); @@ -1202,7 +1289,7 @@ BaseType_t xQueueGenericSendFromISR( QueueHandle_t xQueue, /* MISRA Ref 4.7.1 [Return value shall be checked] */ /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#dir-47 */ /* coverity[misra_c_2012_directive_4_7_violation] */ - uxSavedInterruptStatus = ( UBaseType_t ) taskENTER_CRITICAL_FROM_ISR(); + queueENTER_CRITICAL_FROM_ISR( pxQueue, &uxSavedInterruptStatus ); { if( ( pxQueue->uxMessagesWaiting < pxQueue->uxLength ) || ( xCopyPosition == queueOVERWRITE ) ) { @@ -1233,7 +1320,7 @@ BaseType_t xQueueGenericSendFromISR( QueueHandle_t xQueue, * in the queue has not changed. */ mtCOVERAGE_TEST_MARKER(); } - else if( prvNotifyQueueSetContainer( pxQueue ) != pdFALSE ) + else if( prvNotifyQueueSetContainerFromISR( pxQueue ) != pdFALSE ) { /* The queue is a member of a queue set, and posting * to the queue set caused a higher priority task to @@ -1254,20 +1341,13 @@ BaseType_t xQueueGenericSendFromISR( QueueHandle_t xQueue, } else { - if( listLIST_IS_EMPTY( &( pxQueue->xTasksWaitingToReceive ) ) == pdFALSE ) + if( xTaskRemoveFromEventListFromISR( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) { - if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) + /* The task waiting has a higher priority so + * record that a context switch is required. */ + if( pxHigherPriorityTaskWoken != NULL ) { - /* The task waiting has a higher priority so - * record that a context switch is required. */ - if( pxHigherPriorityTaskWoken != NULL ) - { - *pxHigherPriorityTaskWoken = pdTRUE; - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + *pxHigherPriorityTaskWoken = pdTRUE; } else { @@ -1282,20 +1362,13 @@ BaseType_t xQueueGenericSendFromISR( QueueHandle_t xQueue, } #else /* configUSE_QUEUE_SETS */ { - if( listLIST_IS_EMPTY( &( pxQueue->xTasksWaitingToReceive ) ) == pdFALSE ) + if( xTaskRemoveFromEventListFromISR( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) { - if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) + /* The task waiting has a higher priority so record that a + * context switch is required. */ + if( pxHigherPriorityTaskWoken != NULL ) { - /* The task waiting has a higher priority so record that a - * context switch is required. */ - if( pxHigherPriorityTaskWoken != NULL ) - { - *pxHigherPriorityTaskWoken = pdTRUE; - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + *pxHigherPriorityTaskWoken = pdTRUE; } else { @@ -1327,7 +1400,7 @@ BaseType_t xQueueGenericSendFromISR( QueueHandle_t xQueue, xReturn = errQUEUE_FULL; } } - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + queueEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxQueue ); traceRETURN_xQueueGenericSendFromISR( xReturn ); @@ -1378,7 +1451,7 @@ BaseType_t xQueueGiveFromISR( QueueHandle_t xQueue, /* MISRA Ref 4.7.1 [Return value shall be checked] */ /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#dir-47 */ /* coverity[misra_c_2012_directive_4_7_violation] */ - uxSavedInterruptStatus = ( UBaseType_t ) taskENTER_CRITICAL_FROM_ISR(); + queueENTER_CRITICAL_FROM_ISR( pxQueue, &uxSavedInterruptStatus ); { const UBaseType_t uxMessagesWaiting = pxQueue->uxMessagesWaiting; @@ -1407,7 +1480,7 @@ BaseType_t xQueueGiveFromISR( QueueHandle_t xQueue, { if( pxQueue->pxQueueSetContainer != NULL ) { - if( prvNotifyQueueSetContainer( pxQueue ) != pdFALSE ) + if( prvNotifyQueueSetContainerFromISR( pxQueue ) != pdFALSE ) { /* The semaphore is a member of a queue set, and * posting to the queue set caused a higher priority @@ -1428,20 +1501,13 @@ BaseType_t xQueueGiveFromISR( QueueHandle_t xQueue, } else { - if( listLIST_IS_EMPTY( &( pxQueue->xTasksWaitingToReceive ) ) == pdFALSE ) + if( xTaskRemoveFromEventListFromISR( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) { - if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) + /* The task waiting has a higher priority so + * record that a context switch is required. */ + if( pxHigherPriorityTaskWoken != NULL ) { - /* The task waiting has a higher priority so - * record that a context switch is required. */ - if( pxHigherPriorityTaskWoken != NULL ) - { - *pxHigherPriorityTaskWoken = pdTRUE; - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + *pxHigherPriorityTaskWoken = pdTRUE; } else { @@ -1456,20 +1522,13 @@ BaseType_t xQueueGiveFromISR( QueueHandle_t xQueue, } #else /* configUSE_QUEUE_SETS */ { - if( listLIST_IS_EMPTY( &( pxQueue->xTasksWaitingToReceive ) ) == pdFALSE ) + if( xTaskRemoveFromEventListFromISR( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) { - if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) + /* The task waiting has a higher priority so record that a + * context switch is required. */ + if( pxHigherPriorityTaskWoken != NULL ) { - /* The task waiting has a higher priority so record that a - * context switch is required. */ - if( pxHigherPriorityTaskWoken != NULL ) - { - *pxHigherPriorityTaskWoken = pdTRUE; - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + *pxHigherPriorityTaskWoken = pdTRUE; } else { @@ -1498,7 +1557,7 @@ BaseType_t xQueueGiveFromISR( QueueHandle_t xQueue, xReturn = errQUEUE_FULL; } } - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + queueEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxQueue ); traceRETURN_xQueueGiveFromISR( xReturn ); @@ -1532,7 +1591,7 @@ BaseType_t xQueueReceive( QueueHandle_t xQueue, for( ; ; ) { - taskENTER_CRITICAL(); + queueENTER_CRITICAL( pxQueue ); { const UBaseType_t uxMessagesWaiting = pxQueue->uxMessagesWaiting; @@ -1564,7 +1623,7 @@ BaseType_t xQueueReceive( QueueHandle_t xQueue, mtCOVERAGE_TEST_MARKER(); } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); traceRETURN_xQueueReceive( pdPASS ); @@ -1576,7 +1635,7 @@ BaseType_t xQueueReceive( QueueHandle_t xQueue, { /* The queue was empty and no block time is specified (or * the block time has expired) so leave now. */ - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); traceQUEUE_RECEIVE_FAILED( pxQueue ); traceRETURN_xQueueReceive( errQUEUE_EMPTY ); @@ -1597,13 +1656,12 @@ BaseType_t xQueueReceive( QueueHandle_t xQueue, } } } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); /* Interrupts and other tasks can send to and receive from the queue * now the critical section has been exited. */ - vTaskSuspendAll(); - prvLockQueue( pxQueue ); + queueLOCK( pxQueue ); /* Update the timeout state to see if it has expired yet. */ if( xTaskCheckForTimeOut( &xTimeOut, &xTicksToWait ) == pdFALSE ) @@ -1614,31 +1672,20 @@ BaseType_t xQueueReceive( QueueHandle_t xQueue, { traceBLOCKING_ON_QUEUE_RECEIVE( pxQueue ); vTaskPlaceOnEventList( &( pxQueue->xTasksWaitingToReceive ), xTicksToWait ); - prvUnlockQueue( pxQueue ); - - if( xTaskResumeAll() == pdFALSE ) - { - taskYIELD_WITHIN_API(); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + queueUNLOCK( pxQueue, pdTRUE ); } else { /* The queue contains data again. Loop back to try and read the * data. */ - prvUnlockQueue( pxQueue ); - ( void ) xTaskResumeAll(); + queueUNLOCK( pxQueue, pdFALSE ); } } else { /* Timed out. If there is no data in the queue exit, otherwise loop * back and attempt to read the data. */ - prvUnlockQueue( pxQueue ); - ( void ) xTaskResumeAll(); + queueUNLOCK( pxQueue, pdFALSE ); if( prvIsQueueEmpty( pxQueue ) != pdFALSE ) { @@ -1685,7 +1732,7 @@ BaseType_t xQueueSemaphoreTake( QueueHandle_t xQueue, for( ; ; ) { - taskENTER_CRITICAL(); + queueENTER_CRITICAL( pxQueue ); { /* Semaphores are queues with an item size of 0, and where the * number of messages in the queue is the semaphore's count value. */ @@ -1718,23 +1765,16 @@ BaseType_t xQueueSemaphoreTake( QueueHandle_t xQueue, /* Check to see if other tasks are blocked waiting to give the * semaphore, and if so, unblock the highest priority such task. */ - if( listLIST_IS_EMPTY( &( pxQueue->xTasksWaitingToSend ) ) == pdFALSE ) + if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToSend ) ) != pdFALSE ) { - if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToSend ) ) != pdFALSE ) - { - queueYIELD_IF_USING_PREEMPTION(); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + queueYIELD_IF_USING_PREEMPTION(); } else { mtCOVERAGE_TEST_MARKER(); } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); traceRETURN_xQueueSemaphoreTake( pdPASS ); @@ -1746,7 +1786,7 @@ BaseType_t xQueueSemaphoreTake( QueueHandle_t xQueue, { /* The semaphore count was 0 and no block time is specified * (or the block time has expired) so exit now. */ - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); traceQUEUE_RECEIVE_FAILED( pxQueue ); traceRETURN_xQueueSemaphoreTake( errQUEUE_EMPTY ); @@ -1767,13 +1807,12 @@ BaseType_t xQueueSemaphoreTake( QueueHandle_t xQueue, } } } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); /* Interrupts and other tasks can give to and take from the semaphore * now the critical section has been exited. */ - vTaskSuspendAll(); - prvLockQueue( pxQueue ); + queueLOCK( pxQueue ); /* Update the timeout state to see if it has expired yet. */ if( xTaskCheckForTimeOut( &xTimeOut, &xTicksToWait ) == pdFALSE ) @@ -1800,30 +1839,19 @@ BaseType_t xQueueSemaphoreTake( QueueHandle_t xQueue, #endif /* if ( configUSE_MUTEXES == 1 ) */ vTaskPlaceOnEventList( &( pxQueue->xTasksWaitingToReceive ), xTicksToWait ); - prvUnlockQueue( pxQueue ); - - if( xTaskResumeAll() == pdFALSE ) - { - taskYIELD_WITHIN_API(); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + queueUNLOCK( pxQueue, pdTRUE ); } else { /* There was no timeout and the semaphore count was not 0, so * attempt to take the semaphore again. */ - prvUnlockQueue( pxQueue ); - ( void ) xTaskResumeAll(); + queueUNLOCK( pxQueue, pdFALSE ); } } else { /* Timed out. */ - prvUnlockQueue( pxQueue ); - ( void ) xTaskResumeAll(); + queueUNLOCK( pxQueue, pdFALSE ); /* If the semaphore count is 0 exit now as the timeout has * expired. Otherwise return to attempt to take the semaphore that is @@ -1838,7 +1866,7 @@ BaseType_t xQueueSemaphoreTake( QueueHandle_t xQueue, * test the mutex type again to check it is actually a mutex. */ if( xInheritanceOccurred != pdFALSE ) { - taskENTER_CRITICAL(); + queueENTER_CRITICAL( pxQueue ); { UBaseType_t uxHighestWaitingPriority; @@ -1858,7 +1886,7 @@ BaseType_t xQueueSemaphoreTake( QueueHandle_t xQueue, /* coverity[overrun] */ vTaskPriorityDisinheritAfterTimeout( pxQueue->u.xSemaphore.xMutexHolder, uxHighestWaitingPriority ); } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); } } #endif /* configUSE_MUTEXES */ @@ -1901,7 +1929,7 @@ BaseType_t xQueuePeek( QueueHandle_t xQueue, for( ; ; ) { - taskENTER_CRITICAL(); + queueENTER_CRITICAL( pxQueue ); { const UBaseType_t uxMessagesWaiting = pxQueue->uxMessagesWaiting; @@ -1922,24 +1950,17 @@ BaseType_t xQueuePeek( QueueHandle_t xQueue, /* The data is being left in the queue, so see if there are * any other tasks waiting for the data. */ - if( listLIST_IS_EMPTY( &( pxQueue->xTasksWaitingToReceive ) ) == pdFALSE ) + if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) { - if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToReceive ) ) != pdFALSE ) - { - /* The task waiting has a higher priority than this task. */ - queueYIELD_IF_USING_PREEMPTION(); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + /* The task waiting has a higher priority than this task. */ + queueYIELD_IF_USING_PREEMPTION(); } else { mtCOVERAGE_TEST_MARKER(); } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); traceRETURN_xQueuePeek( pdPASS ); @@ -1951,7 +1972,7 @@ BaseType_t xQueuePeek( QueueHandle_t xQueue, { /* The queue was empty and no block time is specified (or * the block time has expired) so leave now. */ - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); traceQUEUE_PEEK_FAILED( pxQueue ); traceRETURN_xQueuePeek( errQUEUE_EMPTY ); @@ -1973,13 +1994,12 @@ BaseType_t xQueuePeek( QueueHandle_t xQueue, } } } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); /* Interrupts and other tasks can send to and receive from the queue * now that the critical section has been exited. */ - vTaskSuspendAll(); - prvLockQueue( pxQueue ); + queueLOCK( pxQueue ); /* Update the timeout state to see if it has expired yet. */ if( xTaskCheckForTimeOut( &xTimeOut, &xTicksToWait ) == pdFALSE ) @@ -1990,31 +2010,20 @@ BaseType_t xQueuePeek( QueueHandle_t xQueue, { traceBLOCKING_ON_QUEUE_PEEK( pxQueue ); vTaskPlaceOnEventList( &( pxQueue->xTasksWaitingToReceive ), xTicksToWait ); - prvUnlockQueue( pxQueue ); - - if( xTaskResumeAll() == pdFALSE ) - { - taskYIELD_WITHIN_API(); - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + queueUNLOCK( pxQueue, pdTRUE ); } else { /* There is data in the queue now, so don't enter the blocked * state, instead return to try and obtain the data. */ - prvUnlockQueue( pxQueue ); - ( void ) xTaskResumeAll(); + queueUNLOCK( pxQueue, pdFALSE ); } } else { /* The timeout has expired. If there is still no data in the queue * exit, otherwise go back and try to read the data again. */ - prvUnlockQueue( pxQueue ); - ( void ) xTaskResumeAll(); + queueUNLOCK( pxQueue, pdFALSE ); if( prvIsQueueEmpty( pxQueue ) != pdFALSE ) { @@ -2064,7 +2073,7 @@ BaseType_t xQueueReceiveFromISR( QueueHandle_t xQueue, /* MISRA Ref 4.7.1 [Return value shall be checked] */ /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#dir-47 */ /* coverity[misra_c_2012_directive_4_7_violation] */ - uxSavedInterruptStatus = ( UBaseType_t ) taskENTER_CRITICAL_FROM_ISR(); + queueENTER_CRITICAL_FROM_ISR( pxQueue, &uxSavedInterruptStatus ); { const UBaseType_t uxMessagesWaiting = pxQueue->uxMessagesWaiting; @@ -2086,7 +2095,7 @@ BaseType_t xQueueReceiveFromISR( QueueHandle_t xQueue, { if( listLIST_IS_EMPTY( &( pxQueue->xTasksWaitingToSend ) ) == pdFALSE ) { - if( xTaskRemoveFromEventList( &( pxQueue->xTasksWaitingToSend ) ) != pdFALSE ) + if( xTaskRemoveFromEventListFromISR( &( pxQueue->xTasksWaitingToSend ) ) != pdFALSE ) { /* The task waiting has a higher priority than us so * force a context switch. */ @@ -2124,7 +2133,7 @@ BaseType_t xQueueReceiveFromISR( QueueHandle_t xQueue, traceQUEUE_RECEIVE_FROM_ISR_FAILED( pxQueue ); } } - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + queueEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxQueue ); traceRETURN_xQueueReceiveFromISR( xReturn ); @@ -2164,7 +2173,7 @@ BaseType_t xQueuePeekFromISR( QueueHandle_t xQueue, /* MISRA Ref 4.7.1 [Return value shall be checked] */ /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#dir-47 */ /* coverity[misra_c_2012_directive_4_7_violation] */ - uxSavedInterruptStatus = ( UBaseType_t ) taskENTER_CRITICAL_FROM_ISR(); + queueENTER_CRITICAL_FROM_ISR( pxQueue, &uxSavedInterruptStatus ); { /* Cannot block in an ISR, so check there is data available. */ if( pxQueue->uxMessagesWaiting > ( UBaseType_t ) 0 ) @@ -2185,7 +2194,7 @@ BaseType_t xQueuePeekFromISR( QueueHandle_t xQueue, traceQUEUE_PEEK_FROM_ISR_FAILED( pxQueue ); } } - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + queueEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxQueue ); traceRETURN_xQueuePeekFromISR( xReturn ); @@ -2201,11 +2210,11 @@ UBaseType_t uxQueueMessagesWaiting( const QueueHandle_t xQueue ) configASSERT( xQueue ); - portBASE_TYPE_ENTER_CRITICAL(); + queueENTER_CRITICAL( xQueue ); { uxReturn = ( ( Queue_t * ) xQueue )->uxMessagesWaiting; } - portBASE_TYPE_EXIT_CRITICAL(); + queueEXIT_CRITICAL( xQueue ); traceRETURN_uxQueueMessagesWaiting( uxReturn ); @@ -2222,11 +2231,11 @@ UBaseType_t uxQueueSpacesAvailable( const QueueHandle_t xQueue ) configASSERT( pxQueue ); - portBASE_TYPE_ENTER_CRITICAL(); + queueENTER_CRITICAL( pxQueue ); { uxReturn = ( UBaseType_t ) ( pxQueue->uxLength - pxQueue->uxMessagesWaiting ); } - portBASE_TYPE_EXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); traceRETURN_uxQueueSpacesAvailable( uxReturn ); @@ -2492,7 +2501,8 @@ static void prvCopyDataFromQueue( Queue_t * const pxQueue, static void prvUnlockQueue( Queue_t * const pxQueue ) { - /* THIS FUNCTION MUST BE CALLED WITH THE SCHEDULER SUSPENDED. */ + /* THIS FUNCTION MUST BE CALLED WITH THE SCHEDULER SUSPENDED WHEN portUSING_GRANULAR_LOCKS IS 0. + * IT MUST BE CALLED WITH TASK PREEMTION DISABLED WHEN portUSING_GRANULAR_LOCKS IS 1. */ /* The lock counts contains the number of extra data items placed or * removed from the queue while the queue was locked. When a queue is @@ -2630,15 +2640,25 @@ static void prvUnlockQueue( Queue_t * const pxQueue ) pxQueue->cRxLock = queueUNLOCKED; } - taskEXIT_CRITICAL(); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->xISRSpinlock ) ); + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 ) + { + portENABLE_INTERRUPTS(); + } + #else + queueEXIT_CRITICAL( pxQueue ); + #endif } /*-----------------------------------------------------------*/ -static BaseType_t prvIsQueueEmpty( const Queue_t * pxQueue ) +static BaseType_t prvIsQueueEmpty( Queue_t * pxQueue ) { BaseType_t xReturn; - taskENTER_CRITICAL(); + queueENTER_CRITICAL( pxQueue ); { if( pxQueue->uxMessagesWaiting == ( UBaseType_t ) 0 ) { @@ -2649,7 +2669,7 @@ static BaseType_t prvIsQueueEmpty( const Queue_t * pxQueue ) xReturn = pdFALSE; } } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); return xReturn; } @@ -2683,7 +2703,7 @@ static BaseType_t prvIsQueueFull( const Queue_t * pxQueue ) { BaseType_t xReturn; - taskENTER_CRITICAL(); + queueENTER_CRITICAL( pxQueue ); { if( pxQueue->uxMessagesWaiting == pxQueue->uxLength ) { @@ -2694,7 +2714,7 @@ static BaseType_t prvIsQueueFull( const Queue_t * pxQueue ) xReturn = pdFALSE; } } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); return xReturn; } @@ -3174,7 +3194,7 @@ BaseType_t xQueueIsQueueFullFromISR( const QueueHandle_t xQueue ) * time a yield will be performed. If an item is added to the queue while * the queue is locked, and the calling task blocks on the queue, then the * calling task will be immediately unblocked when the queue is unlocked. */ - prvLockQueue( pxQueue ); + queueLOCK( pxQueue ); if( pxQueue->uxMessagesWaiting == ( UBaseType_t ) 0U ) { @@ -3186,7 +3206,7 @@ BaseType_t xQueueIsQueueFullFromISR( const QueueHandle_t xQueue ) mtCOVERAGE_TEST_MARKER(); } - prvUnlockQueue( pxQueue ); + queueUNLOCK( pxQueue, pdFALSE ); traceRETURN_vQueueWaitForMessageRestricted(); } @@ -3238,17 +3258,18 @@ BaseType_t xQueueIsQueueFullFromISR( const QueueHandle_t xQueue ) QueueSetHandle_t xQueueSet ) { BaseType_t xReturn; + Queue_t * const pxQueue = xQueueOrSemaphore; traceENTER_xQueueAddToSet( xQueueOrSemaphore, xQueueSet ); - taskENTER_CRITICAL(); + queueENTER_CRITICAL( pxQueue ); { - if( ( ( Queue_t * ) xQueueOrSemaphore )->pxQueueSetContainer != NULL ) + if( pxQueue->pxQueueSetContainer != NULL ) { /* Cannot add a queue/semaphore to more than one queue set. */ xReturn = pdFAIL; } - else if( ( ( Queue_t * ) xQueueOrSemaphore )->uxMessagesWaiting != ( UBaseType_t ) 0 ) + else if( pxQueue->uxMessagesWaiting != ( UBaseType_t ) 0 ) { /* Cannot add a queue/semaphore to a queue set if there are already * items in the queue/semaphore. */ @@ -3256,11 +3277,11 @@ BaseType_t xQueueIsQueueFullFromISR( const QueueHandle_t xQueue ) } else { - ( ( Queue_t * ) xQueueOrSemaphore )->pxQueueSetContainer = xQueueSet; + pxQueue->pxQueueSetContainer = xQueueSet; xReturn = pdPASS; } } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueue ); traceRETURN_xQueueAddToSet( xReturn ); @@ -3294,12 +3315,12 @@ BaseType_t xQueueIsQueueFullFromISR( const QueueHandle_t xQueue ) } else { - taskENTER_CRITICAL(); + queueENTER_CRITICAL( pxQueueOrSemaphore ); { /* The queue is no longer contained in the set. */ pxQueueOrSemaphore->pxQueueSetContainer = NULL; } - taskEXIT_CRITICAL(); + queueEXIT_CRITICAL( pxQueueOrSemaphore ); xReturn = pdPASS; } @@ -3363,15 +3384,15 @@ BaseType_t xQueueIsQueueFullFromISR( const QueueHandle_t xQueue ) { const BaseType_t xCoreID = portGET_CORE_ID(); - /* This API must be called in a critical section which already has preemption - * and interrupt disabled. */ - portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xTaskSpinlock ) ); + /* This API is called within the queue critical section (which + * already holds the source queue's task spinlock and disables + * interrupts), so only the queue-set container's ISR spinlock + * needs to be acquired here. */ portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xISRSpinlock ) ); { xReturn = prvNotifyQueueSetContainerGeneric( pxQueue, pdFALSE ); } portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xISRSpinlock ) ); - portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xTaskSpinlock ) ); } #endif /* if ( portUSING_GRANULAR_LOCKS == 0 ) */ @@ -3431,17 +3452,21 @@ BaseType_t xQueueIsQueueFullFromISR( const QueueHandle_t xQueue ) if( cTxLock == queueUNLOCKED ) { - if( listLIST_IS_EMPTY( &( pxQueueSetContainer->xTasksWaitingToReceive ) ) == pdFALSE ) + BaseType_t xHigherPriorityTaskWoken; + + if( xIsISR == pdTRUE ) { - if( xTaskRemoveFromEventList( &( pxQueueSetContainer->xTasksWaitingToReceive ) ) != pdFALSE ) - { - /* The task waiting has a higher priority. */ - xReturn = pdTRUE; - } - else - { - mtCOVERAGE_TEST_MARKER(); - } + xHigherPriorityTaskWoken = xTaskRemoveFromEventListFromISR( &( pxQueueSetContainer->xTasksWaitingToReceive ) ); + } + else + { + xHigherPriorityTaskWoken = xTaskRemoveFromEventList( &( pxQueueSetContainer->xTasksWaitingToReceive ) ); + } + + if( xHigherPriorityTaskWoken != pdFALSE ) + { + /* The task waiting has a higher priority. */ + xReturn = pdTRUE; } else { From cbc2f65154e8a11b387b525bd1254960a2ae2217 Mon Sep 17 00:00:00 2001 From: Darian Leung Date: Sun, 16 Jun 2024 01:06:42 +0800 Subject: [PATCH 08/11] change(freertos/smp): Update event_groups.c locking Updated event_groups.c to use granular locking - Added xTaskSpinlock and xISRSpinlock - Replaced critical section macros with data group critical section macros such as taskENTER/EXIT_CRITICAL/_FROM_ISR() with event_groupsENTER/EXIT_CRITICAL/_FROM_ISR(). - Added vEventGroupsEnterCritical/FromISR() and vEventGroupsExitCriti/FromISR() functions that map to the data group critical section macros. - Added prvLockEventGroupForTasks() and prvUnlockEventGroupForTasks() to suspend the event group when executing non-deterministic code. - xEventGroupSetBits() and vEventGroupDelete() accesses the kernel data group directly. Thus, added vTaskSuspendAll()/xTaskResumeAll() to these fucntions. Co-authored-by: Sudeep Mohanty --- event_groups.c | 62 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/event_groups.c b/event_groups.c index ceb3d0d75..f42875456 100644 --- a/event_groups.c +++ b/event_groups.c @@ -63,10 +63,30 @@ #if ( ( configSUPPORT_STATIC_ALLOCATION == 1 ) && ( configSUPPORT_DYNAMIC_ALLOCATION == 1 ) ) uint8_t ucStaticallyAllocated; /**< Set to pdTRUE if the event group is statically allocated to ensure no attempt is made to free the memory. */ #endif + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + portSPINLOCK_TYPE xTaskSpinlock; + portSPINLOCK_TYPE xISRSpinlock; + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ } EventGroup_t; /*-----------------------------------------------------------*/ +/* + * Macros to mark the start and end of a critical code region. + */ + #if ( portUSING_GRANULAR_LOCKS == 1 ) + #define event_groupsENTER_CRITICAL( pxEventBits ) taskDATA_GROUP_ENTER_CRITICAL( &pxEventBits->xTaskSpinlock, &pxEventBits->xISRSpinlock ) + #define event_groupsENTER_CRITICAL_FROM_ISR( pxEventBits, puxSavedInterruptStatus ) taskDATA_GROUP_ENTER_CRITICAL_FROM_ISR( &pxEventBits->xISRSpinlock, puxSavedInterruptStatus ) + #define event_groupsEXIT_CRITICAL( pxEventBits ) taskDATA_GROUP_EXIT_CRITICAL( &pxEventBits->xTaskSpinlock, &pxEventBits->xISRSpinlock ) + #define event_groupsEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxEventBits ) taskDATA_GROUP_EXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, &pxEventBits->xISRSpinlock ) + #else /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + #define event_groupsENTER_CRITICAL( pxEventBits ) taskENTER_CRITICAL(); + #define event_groupsENTER_CRITICAL_FROM_ISR( pxEventBits, puxSavedInterruptStatus ) do { *( puxSavedInterruptStatus ) = taskENTER_CRITICAL_FROM_ISR(); } while( 0 ) + #define event_groupsEXIT_CRITICAL( pxEventBits ) taskEXIT_CRITICAL(); + #define event_groupsEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxEventBits ) taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + /* * Test the bits set in uxCurrentEventBits to see if the wait condition is met. * The wait condition is defined by xWaitForAllBits. If xWaitForAllBits is @@ -108,7 +128,6 @@ /*-----------------------------------------------------------*/ - #if ( configSUPPORT_STATIC_ALLOCATION == 1 ) EventGroupHandle_t xEventGroupCreateStatic( StaticEventGroup_t * pxEventGroupBuffer ) @@ -150,6 +169,13 @@ } #endif /* configSUPPORT_DYNAMIC_ALLOCATION */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + { + portINIT_SPINLOCK( &( pxEventBits->xTaskSpinlock ) ); + portINIT_SPINLOCK( &( pxEventBits->xISRSpinlock ) ); + } + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceEVENT_GROUP_CREATE( pxEventBits ); } else @@ -195,6 +221,13 @@ } #endif /* configSUPPORT_STATIC_ALLOCATION */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + { + portINIT_SPINLOCK( &( pxEventBits->xTaskSpinlock ) ); + portINIT_SPINLOCK( &( pxEventBits->xISRSpinlock ) ); + } + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceEVENT_GROUP_CREATE( pxEventBits ); } else @@ -230,7 +263,7 @@ } #endif - vTaskSuspendAll(); + event_groupsLOCK( pxEventBits ); { uxOriginalBitValue = pxEventBits->uxEventBits; @@ -295,7 +328,7 @@ if( ( uxReturn & eventUNBLOCKED_DUE_TO_BIT_SET ) == ( EventBits_t ) 0 ) { /* The task timed out, just return the current event bit value. */ - taskENTER_CRITICAL(); + event_groupsENTER_CRITICAL( pxEventBits ); { uxReturn = pxEventBits->uxEventBits; @@ -312,7 +345,7 @@ mtCOVERAGE_TEST_MARKER(); } } - taskEXIT_CRITICAL(); + event_groupsEXIT_CRITICAL( pxEventBits ); xTimeoutOccurred = pdTRUE; } @@ -361,7 +394,7 @@ } #endif - vTaskSuspendAll(); + event_groupsLOCK( pxEventBits ); { const EventBits_t uxCurrentEventBits = pxEventBits->uxEventBits; @@ -450,7 +483,7 @@ if( ( uxReturn & eventUNBLOCKED_DUE_TO_BIT_SET ) == ( EventBits_t ) 0 ) { - taskENTER_CRITICAL(); + event_groupsENTER_CRITICAL( pxEventBits ); { /* The task timed out, just return the current event bit value. */ uxReturn = pxEventBits->uxEventBits; @@ -475,7 +508,7 @@ xTimeoutOccurred = pdTRUE; } - taskEXIT_CRITICAL(); + event_groupsEXIT_CRITICAL( pxEventBits ); } else { @@ -510,7 +543,7 @@ configASSERT( xEventGroup ); configASSERT( ( uxBitsToClear & eventEVENT_BITS_CONTROL_BYTES ) == 0 ); - taskENTER_CRITICAL(); + event_groupsENTER_CRITICAL( pxEventBits ); { traceEVENT_GROUP_CLEAR_BITS( xEventGroup, uxBitsToClear ); @@ -521,7 +554,7 @@ /* Clear the bits. */ pxEventBits->uxEventBits &= ~uxBitsToClear; } - taskEXIT_CRITICAL(); + event_groupsEXIT_CRITICAL( pxEventBits ); traceRETURN_xEventGroupClearBits( uxReturn ); @@ -552,7 +585,7 @@ EventBits_t xEventGroupGetBitsFromISR( EventGroupHandle_t xEventGroup ) { UBaseType_t uxSavedInterruptStatus; - EventGroup_t const * const pxEventBits = xEventGroup; + EventGroup_t * const pxEventBits = xEventGroup; EventBits_t uxReturn; traceENTER_xEventGroupGetBitsFromISR( xEventGroup ); @@ -560,11 +593,11 @@ /* MISRA Ref 4.7.1 [Return value shall be checked] */ /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#dir-47 */ /* coverity[misra_c_2012_directive_4_7_violation] */ - uxSavedInterruptStatus = taskENTER_CRITICAL_FROM_ISR(); + event_groupsENTER_CRITICAL_FROM_ISR( pxEventBits, &uxSavedInterruptStatus ); { uxReturn = pxEventBits->uxEventBits; } - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + event_groupsEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxEventBits ); traceRETURN_xEventGroupGetBitsFromISR( uxReturn ); @@ -592,7 +625,7 @@ pxList = &( pxEventBits->xTasksWaitingForBits ); pxListEnd = listGET_END_MARKER( pxList ); - vTaskSuspendAll(); + event_groupsLOCK( pxEventBits ); { traceEVENT_GROUP_SET_BITS( xEventGroup, uxBitsToSet ); @@ -697,7 +730,7 @@ pxTasksWaitingForBits = &( pxEventBits->xTasksWaitingForBits ); - vTaskSuspendAll(); + event_groupsLOCK( pxEventBits ); { traceEVENT_GROUP_DELETE( xEventGroup ); @@ -825,6 +858,7 @@ } /*-----------------------------------------------------------*/ + static BaseType_t prvTestWaitCondition( const EventBits_t uxCurrentEventBits, const EventBits_t uxBitsToWaitFor, const BaseType_t xWaitForAllBits ) From 3ba8783ca407fc93ce905be259f645efa47c13e7 Mon Sep 17 00:00:00 2001 From: Darian Leung Date: Sun, 16 Jun 2024 01:11:43 +0800 Subject: [PATCH 09/11] change(freertos/smp): Update stream_buffer.c locking Updated stream_buffer.c to use granular locking - Added xTaskSpinlock and xISRSpinlock - Replaced critical section macros with data group critical section macros such as taskENTER/EXIT_CRITICAL/_FROM_ISR() with sbENTER/EXIT_CRITICAL_FROM_ISR(). - Added vStreambuffersEnterCritical/FromISR() and vStreambuffersExitCritical/FromISR() to map to the data group critical section macros. - Added prvLockStreamBufferForTasks() and prvUnlockStreamBufferForTasks() to suspend the stream buffer when executing non-deterministic code. Co-authored-by: Sudeep Mohanty --- stream_buffer.c | 114 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 93 insertions(+), 21 deletions(-) diff --git a/stream_buffer.c b/stream_buffer.c index a4c6d268d..1f24ebdd6 100644 --- a/stream_buffer.c +++ b/stream_buffer.c @@ -58,6 +58,39 @@ #error INCLUDE_xTaskGetCurrentTaskHandle must be set to 1 to build stream_buffer.c #endif + +/* + * Macros to mark the start and end of a critical code region. + */ + #if ( portUSING_GRANULAR_LOCKS == 1 ) + #define sbENTER_CRITICAL( pxStreamBuffer ) taskDATA_GROUP_ENTER_CRITICAL( &( ( pxStreamBuffer )->xTaskSpinlock ), &( ( pxStreamBuffer )->xISRSpinlock ) ) + #define sbENTER_CRITICAL_FROM_ISR( pxStreamBuffer, puxSavedInterruptStatus ) taskDATA_GROUP_ENTER_CRITICAL_FROM_ISR( &( ( pxStreamBuffer )->xISRSpinlock ), puxSavedInterruptStatus ) + #define sbEXIT_CRITICAL( pxStreamBuffer ) taskDATA_GROUP_EXIT_CRITICAL( &( ( pxStreamBuffer )->xTaskSpinlock ), &( ( pxStreamBuffer )->xISRSpinlock ) ) + #define sbEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxStreamBuffer ) taskDATA_GROUP_EXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, &( ( pxStreamBuffer )->xISRSpinlock ) ) + #else /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + #define sbENTER_CRITICAL( pxStreamBuffer ) taskENTER_CRITICAL() + #define sbENTER_CRITICAL_FROM_ISR( pxStreamBuffer, puxSavedInterruptStatus ) do { *( puxSavedInterruptStatus ) = taskENTER_CRITICAL_FROM_ISR(); } while( 0 ) + #define sbEXIT_CRITICAL( pxStreamBuffer ) taskEXIT_CRITICAL() + #define sbEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxStreamBuffer ) taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) + #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + +/* + * Macro used to lock and unlock a stream buffer. When a task locks a stream + * buffer, the task will have thread safe non-deterministic access to the stream + * buffer. + * - Concurrent access from other tasks will be blocked by the xTaskSpinlock + * - Concurrent access from ISRs will be pended + * + * When the task unlocks the stream buffer, all pended access attempts are handled. + */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #define sbLOCK( pxStreamBuffer ) taskDATA_GROUP_LOCK( &( ( pxStreamBuffer )->xTaskSpinlock ) ) + #define sbUNLOCK( pxStreamBuffer ) taskDATA_GROUP_UNLOCK( &( ( pxStreamBuffer )->xTaskSpinlock ) ) + #else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #define sbLOCK( pxStreamBuffer ) vTaskSuspendAll() + #define sbUNLOCK( pxStreamBuffer ) ( void ) xTaskResumeAll() + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + /* If the user has not provided application specific Rx notification macros, * or #defined the notification macros away, then provide default implementations * that uses task notifications. */ @@ -65,7 +98,7 @@ #define sbRECEIVE_COMPLETED( pxStreamBuffer ) \ do \ { \ - vTaskSuspendAll(); \ + sbLOCK( pxStreamBuffer ); \ { \ if( ( pxStreamBuffer )->xTaskWaitingToSend != NULL ) \ { \ @@ -76,7 +109,7 @@ ( pxStreamBuffer )->xTaskWaitingToSend = NULL; \ } \ } \ - ( void ) xTaskResumeAll(); \ + sbUNLOCK( pxStreamBuffer ); \ } while( 0 ) #endif /* sbRECEIVE_COMPLETED */ @@ -105,7 +138,7 @@ do { \ UBaseType_t uxSavedInterruptStatus; \ \ - uxSavedInterruptStatus = taskENTER_CRITICAL_FROM_ISR(); \ + sbENTER_CRITICAL_FROM_ISR( pxStreamBuffer, &uxSavedInterruptStatus ); \ { \ if( ( pxStreamBuffer )->xTaskWaitingToSend != NULL ) \ { \ @@ -117,7 +150,7 @@ ( pxStreamBuffer )->xTaskWaitingToSend = NULL; \ } \ } \ - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); \ + sbEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxStreamBuffer ); \ } while( 0 ) #endif /* sbRECEIVE_COMPLETED_FROM_ISR */ @@ -145,7 +178,7 @@ */ #ifndef sbSEND_COMPLETED #define sbSEND_COMPLETED( pxStreamBuffer ) \ - vTaskSuspendAll(); \ + sbLOCK( pxStreamBuffer ); \ { \ if( ( pxStreamBuffer )->xTaskWaitingToReceive != NULL ) \ { \ @@ -156,7 +189,7 @@ ( pxStreamBuffer )->xTaskWaitingToReceive = NULL; \ } \ } \ - ( void ) xTaskResumeAll() + sbUNLOCK( pxStreamBuffer ) #endif /* sbSEND_COMPLETED */ /* If user has provided a per-instance send completed callback, then @@ -184,7 +217,7 @@ do { \ UBaseType_t uxSavedInterruptStatus; \ \ - uxSavedInterruptStatus = taskENTER_CRITICAL_FROM_ISR(); \ + sbENTER_CRITICAL_FROM_ISR( pxStreamBuffer, &uxSavedInterruptStatus ); \ { \ if( ( pxStreamBuffer )->xTaskWaitingToReceive != NULL ) \ { \ @@ -196,7 +229,7 @@ ( pxStreamBuffer )->xTaskWaitingToReceive = NULL; \ } \ } \ - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); \ + sbEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxStreamBuffer ); \ } while( 0 ) #endif /* sbSEND_COMPLETE_FROM_ISR */ @@ -249,6 +282,10 @@ typedef struct StreamBufferDef_t StreamBufferCallbackFunction_t pxReceiveCompletedCallback; /* Optional callback called on receive complete. sbRECEIVE_COMPLETED is called if this is NULL. */ #endif UBaseType_t uxNotificationIndex; /* The index we are using for notification, by default tskDEFAULT_INDEX_TO_NOTIFY. */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + portSPINLOCK_TYPE xTaskSpinlock; + portSPINLOCK_TYPE xISRSpinlock; + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ } StreamBuffer_t; /* @@ -336,6 +373,7 @@ static void prvInitialiseNewStreamBuffer( StreamBuffer_t * const pxStreamBuffer, StreamBufferCallbackFunction_t pxReceiveCompletedCallback ) PRIVILEGED_FUNCTION; /*-----------------------------------------------------------*/ + #if ( configSUPPORT_DYNAMIC_ALLOCATION == 1 ) StreamBufferHandle_t xStreamBufferGenericCreate( size_t xBufferSizeBytes, size_t xTriggerLevelBytes, @@ -414,6 +452,13 @@ static void prvInitialiseNewStreamBuffer( StreamBuffer_t * const pxStreamBuffer, pxSendCompletedCallback, pxReceiveCompletedCallback ); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + { + portINIT_SPINLOCK( &( ( ( StreamBuffer_t * ) pvAllocatedMemory )->xTaskSpinlock ) ); + portINIT_SPINLOCK( &( ( ( StreamBuffer_t * ) pvAllocatedMemory )->xISRSpinlock ) ); + } + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceSTREAM_BUFFER_CREATE( ( ( StreamBuffer_t * ) pvAllocatedMemory ), xStreamBufferType ); } else @@ -508,6 +553,13 @@ static void prvInitialiseNewStreamBuffer( StreamBuffer_t * const pxStreamBuffer, * again. */ pxStreamBuffer->ucFlags |= sbFLAGS_IS_STATICALLY_ALLOCATED; + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + { + portINIT_SPINLOCK( &( pxStreamBuffer->xTaskSpinlock ) ); + portINIT_SPINLOCK( &( pxStreamBuffer->xISRSpinlock ) ); + } + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + traceSTREAM_BUFFER_CREATE( pxStreamBuffer, xStreamBufferType ); /* MISRA Ref 11.3.1 [Misaligned access] */ @@ -623,7 +675,7 @@ BaseType_t xStreamBufferReset( StreamBufferHandle_t xStreamBuffer ) #endif /* Can only reset a message buffer if there are no tasks blocked on it. */ - taskENTER_CRITICAL(); + sbENTER_CRITICAL( pxStreamBuffer ); { if( ( pxStreamBuffer->xTaskWaitingToReceive == NULL ) && ( pxStreamBuffer->xTaskWaitingToSend == NULL ) ) { @@ -653,7 +705,7 @@ BaseType_t xStreamBufferReset( StreamBufferHandle_t xStreamBuffer ) xReturn = pdPASS; } } - taskEXIT_CRITICAL(); + sbEXIT_CRITICAL( pxStreamBuffer ); traceRETURN_xStreamBufferReset( xReturn ); @@ -881,7 +933,7 @@ size_t xStreamBufferSend( StreamBufferHandle_t xStreamBuffer, { /* Wait until the required number of bytes are free in the message * buffer. */ - taskENTER_CRITICAL(); + sbENTER_CRITICAL( pxStreamBuffer ); { xSpace = xStreamBufferSpacesAvailable( pxStreamBuffer ); @@ -896,15 +948,19 @@ size_t xStreamBufferSend( StreamBufferHandle_t xStreamBuffer, } else { - taskEXIT_CRITICAL(); + sbEXIT_CRITICAL( pxStreamBuffer ); break; } } - taskEXIT_CRITICAL(); + sbEXIT_CRITICAL( pxStreamBuffer ); traceBLOCKING_ON_STREAM_BUFFER_SEND( xStreamBuffer ); ( void ) xTaskNotifyWaitIndexed( pxStreamBuffer->uxNotificationIndex, ( uint32_t ) 0, ( uint32_t ) 0, NULL, xTicksToWait ); - pxStreamBuffer->xTaskWaitingToSend = NULL; + sbENTER_CRITICAL( pxStreamBuffer ); + { + pxStreamBuffer->xTaskWaitingToSend = NULL; + } + sbEXIT_CRITICAL( pxStreamBuffer ); } while( xTaskCheckForTimeOut( &xTimeOut, &xTicksToWait ) == pdFALSE ); } else @@ -1099,7 +1155,7 @@ size_t xStreamBufferReceive( StreamBufferHandle_t xStreamBuffer, { /* Checking if there is data and clearing the notification state must be * performed atomically. */ - taskENTER_CRITICAL(); + sbENTER_CRITICAL( pxStreamBuffer ); { xBytesAvailable = prvBytesInBuffer( pxStreamBuffer ); @@ -1124,14 +1180,18 @@ size_t xStreamBufferReceive( StreamBufferHandle_t xStreamBuffer, mtCOVERAGE_TEST_MARKER(); } } - taskEXIT_CRITICAL(); + sbEXIT_CRITICAL( pxStreamBuffer ); if( xBytesAvailable <= xBytesToStoreMessageLength ) { /* Wait for data to be available. */ traceBLOCKING_ON_STREAM_BUFFER_RECEIVE( xStreamBuffer ); ( void ) xTaskNotifyWaitIndexed( pxStreamBuffer->uxNotificationIndex, ( uint32_t ) 0, ( uint32_t ) 0, NULL, xTicksToWait ); - pxStreamBuffer->xTaskWaitingToReceive = NULL; + sbENTER_CRITICAL( pxStreamBuffer ); + { + pxStreamBuffer->xTaskWaitingToReceive = NULL; + } + sbEXIT_CRITICAL( pxStreamBuffer ); /* Recheck the data available after blocking. */ xBytesAvailable = prvBytesInBuffer( pxStreamBuffer ); @@ -1421,7 +1481,7 @@ BaseType_t xStreamBufferSendCompletedFromISR( StreamBufferHandle_t xStreamBuffer /* MISRA Ref 4.7.1 [Return value shall be checked] */ /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#dir-47 */ /* coverity[misra_c_2012_directive_4_7_violation] */ - uxSavedInterruptStatus = taskENTER_CRITICAL_FROM_ISR(); + sbENTER_CRITICAL_FROM_ISR( pxStreamBuffer, &uxSavedInterruptStatus ); { if( ( pxStreamBuffer )->xTaskWaitingToReceive != NULL ) { @@ -1438,7 +1498,7 @@ BaseType_t xStreamBufferSendCompletedFromISR( StreamBufferHandle_t xStreamBuffer xReturn = pdFALSE; } } - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + sbEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxStreamBuffer ); traceRETURN_xStreamBufferSendCompletedFromISR( xReturn ); @@ -1460,7 +1520,7 @@ BaseType_t xStreamBufferReceiveCompletedFromISR( StreamBufferHandle_t xStreamBuf /* MISRA Ref 4.7.1 [Return value shall be checked] */ /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#dir-47 */ /* coverity[misra_c_2012_directive_4_7_violation] */ - uxSavedInterruptStatus = taskENTER_CRITICAL_FROM_ISR(); + sbENTER_CRITICAL_FROM_ISR( pxStreamBuffer, &uxSavedInterruptStatus ); { if( ( pxStreamBuffer )->xTaskWaitingToSend != NULL ) { @@ -1477,7 +1537,7 @@ BaseType_t xStreamBufferReceiveCompletedFromISR( StreamBufferHandle_t xStreamBuf xReturn = pdFALSE; } } - taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + sbEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxStreamBuffer ); traceRETURN_xStreamBufferReceiveCompletedFromISR( xReturn ); @@ -1633,6 +1693,13 @@ static void prvInitialiseNewStreamBuffer( StreamBuffer_t * const pxStreamBuffer, StreamBufferCallbackFunction_t pxSendCompletedCallback, StreamBufferCallbackFunction_t pxReceiveCompletedCallback ) { + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + /* Preserve the embedded spinlocks across the memset; the reset + * path runs with xTaskSpinlock held by the caller. */ + portSPINLOCK_TYPE xTaskSpinlock = pxStreamBuffer->xTaskSpinlock; + portSPINLOCK_TYPE xISRSpinlock = pxStreamBuffer->xISRSpinlock; + #endif + /* Assert here is deliberately writing to the entire buffer to ensure it can * be written to without generating exceptions, and is setting the buffer to a * known value to assist in development/debugging. */ @@ -1652,6 +1719,11 @@ static void prvInitialiseNewStreamBuffer( StreamBuffer_t * const pxStreamBuffer, pxStreamBuffer->xTriggerLevelBytes = xTriggerLevelBytes; pxStreamBuffer->ucFlags = ucFlags; pxStreamBuffer->uxNotificationIndex = tskDEFAULT_INDEX_TO_NOTIFY; + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + pxStreamBuffer->xTaskSpinlock = xTaskSpinlock; + pxStreamBuffer->xISRSpinlock = xISRSpinlock; + #endif #if ( configUSE_SB_COMPLETED_CALLBACK == 1 ) { pxStreamBuffer->pxSendCompletedCallback = pxSendCompletedCallback; From a85f9cfaba097641ff65cc44cda823a27d655ab9 Mon Sep 17 00:00:00 2001 From: Darian Leung Date: Sun, 16 Jun 2024 01:13:43 +0800 Subject: [PATCH 10/11] change(freertos/smp): Update timers.c locking Updated timers.c to use granular locking - Added xTaskSpinlock and xISRSpinlock - Replaced critical section macros with data group critical section macros such as taskENTER/EXIT_CRITICAL() with tmrENTER/EXIT_CRITICAL(). - Added vTimerEnterCritical() and vTimerExitCritical() to map to the data group critical section macros. Co-authored-by: Sudeep Mohanty --- timers.c | 67 ++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/timers.c b/timers.c index b283522cb..53097568c 100644 --- a/timers.c +++ b/timers.c @@ -79,6 +79,17 @@ #define tmrSTATUS_IS_STATICALLY_ALLOCATED ( 0x02U ) #define tmrSTATUS_IS_AUTORELOAD ( 0x04U ) +/* + * Macros to mark the start and end of a critical code region. + */ + #if ( portUSING_GRANULAR_LOCKS == 1 ) + #define tmrENTER_CRITICAL() taskDATA_GROUP_ENTER_CRITICAL( &xTimerTaskSpinlock, &xTimerISRSpinlock ) + #define tmrEXIT_CRITICAL() taskDATA_GROUP_EXIT_CRITICAL( &xTimerTaskSpinlock, &xTimerISRSpinlock ) + #else /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + #define tmrENTER_CRITICAL() taskENTER_CRITICAL() + #define tmrEXIT_CRITICAL() taskEXIT_CRITICAL() + #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + /* The definition of the timers themselves. */ typedef struct tmrTimerControl /* The old naming convention is used to prevent breaking kernel aware debuggers. */ { @@ -582,7 +593,7 @@ traceENTER_vTimerSetReloadMode( xTimer, xAutoReload ); configASSERT( xTimer ); - taskENTER_CRITICAL(); + tmrENTER_CRITICAL(); { if( xAutoReload != pdFALSE ) { @@ -593,7 +604,7 @@ pxTimer->ucStatus &= ( ( uint8_t ) ~tmrSTATUS_IS_AUTORELOAD ); } } - taskEXIT_CRITICAL(); + tmrEXIT_CRITICAL(); traceRETURN_vTimerSetReloadMode(); } @@ -607,7 +618,15 @@ traceENTER_xTimerGetReloadMode( xTimer ); configASSERT( xTimer ); - portBASE_TYPE_ENTER_CRITICAL(); + #if ( ( configNUMBER_OF_CORES > 1 ) ) + { + tmrENTER_CRITICAL(); + } + #else + { + portBASE_TYPE_ENTER_CRITICAL(); + } + #endif { if( ( pxTimer->ucStatus & tmrSTATUS_IS_AUTORELOAD ) == 0U ) { @@ -620,7 +639,15 @@ xReturn = pdTRUE; } } - portBASE_TYPE_EXIT_CRITICAL(); + #if ( ( configNUMBER_OF_CORES > 1 ) ) + { + tmrEXIT_CRITICAL(); + } + #else + { + portBASE_TYPE_EXIT_CRITICAL(); + } + #endif traceRETURN_xTimerGetReloadMode( xReturn ); @@ -1126,7 +1153,7 @@ /* Check that the list from which active timers are referenced, and the * queue used to communicate with the timer service, have been * initialised. */ - taskENTER_CRITICAL(); + tmrENTER_CRITICAL(); { if( xTimerQueue == NULL ) { @@ -1168,7 +1195,7 @@ mtCOVERAGE_TEST_MARKER(); } } - taskEXIT_CRITICAL(); + tmrEXIT_CRITICAL(); } /*-----------------------------------------------------------*/ @@ -1182,7 +1209,15 @@ configASSERT( xTimer ); /* Is the timer in the list of active timers? */ - portBASE_TYPE_ENTER_CRITICAL(); + #if ( ( configNUMBER_OF_CORES > 1 ) ) + { + tmrENTER_CRITICAL(); + } + #else + { + portBASE_TYPE_ENTER_CRITICAL(); + } + #endif { if( ( pxTimer->ucStatus & tmrSTATUS_IS_ACTIVE ) == 0U ) { @@ -1193,7 +1228,15 @@ xReturn = pdTRUE; } } - portBASE_TYPE_EXIT_CRITICAL(); + #if ( ( configNUMBER_OF_CORES > 1 ) ) + { + tmrEXIT_CRITICAL(); + } + #else + { + portBASE_TYPE_EXIT_CRITICAL(); + } + #endif traceRETURN_xTimerIsTimerActive( xReturn ); @@ -1210,11 +1253,11 @@ configASSERT( xTimer ); - taskENTER_CRITICAL(); + tmrENTER_CRITICAL(); { pvReturn = pxTimer->pvTimerID; } - taskEXIT_CRITICAL(); + tmrEXIT_CRITICAL(); traceRETURN_pvTimerGetTimerID( pvReturn ); @@ -1231,11 +1274,11 @@ configASSERT( xTimer ); - taskENTER_CRITICAL(); + tmrENTER_CRITICAL(); { pxTimer->pvTimerID = pvNewID; } - taskEXIT_CRITICAL(); + tmrEXIT_CRITICAL(); traceRETURN_vTimerSetTimerID(); } From 5aede4aeaaceb18bca5bdb99b89faf07cc66d15b Mon Sep 17 00:00:00 2001 From: Darian Leung Date: Mon, 17 Jun 2024 15:23:45 +0800 Subject: [PATCH 11/11] docs(freertos/smp): Add granular locks design document The design document captures the **Dual Spinlock With Data Group Locking** scheme used by the granular-locks implementation: data-group definitions and hierarchy, public and port-layer API, TCB locks, deferred state changes, ISR-only kernel critical-section helpers, event-list TOCTOU considerations, and current limitations. --- freertos_smp_granular_locks_design.md | 442 ++++++++++++++++++++++++++ 1 file changed, 442 insertions(+) create mode 100644 freertos_smp_granular_locks_design.md diff --git a/freertos_smp_granular_locks_design.md b/freertos_smp_granular_locks_design.md new file mode 100644 index 000000000..3f785d4a5 --- /dev/null +++ b/freertos_smp_granular_locks_design.md @@ -0,0 +1,442 @@ +# FreeRTOS SMP Granular Locks — Design + +## Introduction + +Historically, the SMP FreeRTOS kernel has implemented critical sections using a "Global Locking" approach, where all kernel data is protected by a single pair of spinlocks (the Task Lock and ISR Lock). Every critical section contests for this pair of spinlocks, even when the sections protect unrelated data. + +This document describes the **"Dual Spinlock With Data Group Locking"** scheme that replaces global locking with per-data-group locks. The goal is to reduce contention between concurrent operations on independent kernel objects (queues, event groups, stream buffers, etc.) so that the SMP kernel scales better with the number of cores. + +Granular locking is opt-in via the new port-layer configuration `portUSING_GRANULAR_LOCKS`. With it disabled the kernel behaves exactly as before; with it enabled the per-data-group locks described below are used. + +Source-code changes are layered on the V11.1.0 release of the upstream FreeRTOS kernel. + +### Status (as of this branch) + +- Implemented in the kernel against `configNUMBER_OF_CORES > 1` builds. +- Verified on the Espressif ESP32 (Xtensa, dual-core) target via the ESP-IDF FreeRTOS unit-test suite. +- Other ports remain to be updated; reference port macros and the porting checklist are listed in the *Porting Interface* section. + +# Data Groups + +To make the spinlocks granular, FreeRTOS data will be organized into the following data groups, where each data group is protected by their own set of spinlocks. + +- Kernel Data Group + - All data in `tasks.c` and all event lists (e.g., `xTasksWaitingToSend` and `xTasksWaitingToReceive` in the queue objects) +- Queue Data Group + - Each queue object (`Queue_t`) is its own data group (excluding the task lists) +- Event Group Data Group + - Each event group object (`EventGroup_t`) is its own data group (excluding the task lists) +- Stream Buffer Data Group + - Each stream buffer object (`StreamBuffer_t`) is its own data group (excluding the task lists) +- Timers + - All data in `timers.c` and timer objects, belong to the same Timer Data Group +- User/Port Data Groups + - The user and ports are free to organize their own data in to data groups of their choosing + +# Dual Spinlock With Data Group Locking + +The **"Dual Spinlock With Data Group Locking"** uses a pair of spinlocks to protect each data group (namely the `xTaskSpinlock` and `xISRSpinlock` spinlocks). + +```c +typedef struct +{ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + portSPINLOCK_TYPE xTaskSpinlock; + portSPINLOCK_TYPE xISRSpinlock; + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ +} xSomeDataGroup_t; +``` + +However, each data group must also allow for non-deterministic access with interrupts **enabled** by providing a pair of lock/unlock functions. These functions must block access to other tasks trying to access members of a data group and must pend access to ISRs trying to access members of the data group. + +```c +static void prvLockSomeDataGroupForTasks( xSomeDataGroup_t * const pxSomDataGroup ); +static void prvUnlockSomeDataGroupForTasks( xSomeDataGroup_t * const pxSomDataGroup ); +``` + +In simple terms, the "Dual Spinlock With Data Group Locking" is an extension is the existing dual spinlock (i.e., Task and ISR spinlock) approach used in the SMP FreeRTOS kernel, but replicated across different data groups. + +## Data Group Critical Sections (Granular Locks) + +A critical section for a data group can be achieved as follows: + +- When entering a data group critical section from a task + 1. Disable interrupts + 2. Take `xTaskSpinlock` of data group + 3. Take `xISRSpinlock` of data group + 4. Increment nesting count +- When entering data group critical section from an ISR + 1. Disable interrupts + 2. Take `xISRSpinlock` of data group + 3. Increment nesting count + +When exiting a data group critical section, the procedure is reversed. Furthermore, since yield requests are pended when inside a critical section, exiting a task critical section will also need to handle any pended yields. + +- When exiting a data group critical section from a task + 1. Release `xISRSpinlock` of data group + 2. Release `xTaskSpinlock` of data group + 3. Decrement nesting count + 4. Reenable interrupts if nesting count is 0 + 5. Trigger yield if there is a yield pending +- When exiting data group critical section form an ISR + 1. Release `xISRSpinlock` of data group + 2. Decrement nesting count + 3. Reenable interrupts if nesting count is 0 + +Entering multiple data group critical sections in a nested manner is permitted. This means, if a code path has already entered a critical section in data group A, it can then enter a critical section in data group B. This is analogous to nested critical sections. However, care must be taken to avoid deadlocks. This can be achieved by organizing data groups into a hierarchy, where a higher layer data group cannot nest into a lower one. + +``` + +-------------------+ + | Kernel Data Group | + +-------------------+ + + +-------------------+ +---------------------------+ +--------------------------+ +------------------+ + | Queue Data Groups | | Stream Buffer Data Groups | | Event Groups Data Groups | | Timer Data Group | + +-------------------+ +---------------------------+ +--------------------------+ +------------------+ + ++------------------------------------------------------------------------------------------------------+ +| User Data Groups | ++------------------------------------------------------------------------------------------------------+ +``` + +If nested locking only occurs from bottom up (e.g., User data group can nested into a Queue data group which in turn can nested into Kernel data group), then deadlocking will never occur. + +## Data Group Locking + +FreeRTOS does not permit walking linked lists while interrupts are disabled to ensure deterministic ISR latency. Therefore, each data group must provide a method of locking so that non-deterministic operations can be executed for a data group. While a data group is locked: + +- Preemption is disabled for the current task +- Interrupts remained enabled +- The data group's `xTaskSpinlock` is taken to prevent tasks running on other cores from accessing the data group +- Any ISR that attempts to update the data group will have their access pended. These pended accesses will be handled on resumption + +The logic of suspending a data group is analogous to the logic of `vTaskSuspendAll()`/`xTaskResumeAll()` and `prvLockQueue()`/`prvUnlockQueue()` in the existing SMP kernel. + +The details of how ISR accesses are pended during suspension will be specific to each data group type, thus the implementation of the suspend/resumption functions also be specified to each data group type. However, the procedure for data group suspension and resumption will typically be as follows: + +- Suspension + 1. Disable preemption + 2. Lock the data group + 3. Set a suspension flag that indicates the data group is suspended + 4. Unlock the data group, but keep holding `xTaskSpinlock` +- Resumption + 1. Lock the data group + 2. Clear the suspension flag + 3. Handle all pended accesses from ISRs + 4. Unlock the data group, thus releasing `xTaskSpinlock` + +Locking multiple data groups in a nested manner is permitted, meaning if a code path has already locked data group A, it can then lock data group B. This is analogous to nested `vTaskSuspendAll()` calls. Similar to data group locking, deadlocks can be avoided by organizing data groups into a hierarchy. + +## Thread Safety Check + +Under SMP, there are four sources of concurrent access for a particular data group: + +- Preempting task on the same core +- Preempting ISR on the same core +- Concurrent task on another core +- Concurrent ISR on another core + +This section checks that the data group critical section and locking mechanisms mentioned, ensure thread safety from each concurrent source of access. + +- Data Group Critical Section from tasks: Interrupts are disabled, `xTaskSpinlock` and `xISRSpinlock` are taken + - Task (same core): Context switch cannot occur because interrupts are disabled + - Task (other cores): The task will spin on `xTaskSpinlock` + - ISR (same core): Interrupts on the current core are disabled + - ISR (other cores): ISR will spin on `xISRSpinlock` + +- Data Group Critical Sections from ISRs: Interrupts are disabled, `xISRSpinlock` is taken + - Task (same core): Context switch cannot occur because we are in an ISR + - Task (other cores): The task will spin on `xISRSpinlock` + - ISR (same core): Interrupts on the current core are disabled + - ISR (other cores): ISR will spin on `xISRSpinlock` + +- Data Group Locking from tasks: Preemption is disabled, `xTaskSpinlock` is taken + - Task (same core): Context switch cannot occur because preemption is disabled + - Task (other cores): The task will spin on `xTaskSpinlock` + - ISR (same core): Critical section is entered because `xISRSpinlock` is not held, but access is pended + - ISR (other cores): Critical section is entered because `xISRSpinlock` is not held, but access is pended + +# Public API Changes + +To support **"Dual Spinlock With Data Group Locking"**, the following changes have been made to the public facing API. These changes are non-breaking, meaning that applications that can build against the existing SMP FreeRTOS kernel will still be able to build even with granular locking enabled (albeit less performant). + +The following APIs have been added to enter/exit a critical section in a data group. This are called by FreeRTOS source code to mark critical sections in data groups. However, users can also create their own data groups and enter/exit critical sections in the same manner. + +If granular locking is disabled, these macros simply revert to being the standard task enter/exit critical macros. + +```c +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define data_groupENTER_CRITICAL() portENTER_CRITICAL_DATA_GROUP( ( portSPINLOCK_TYPE * ) pxTaskSpinlock, ( portSPINLOCK_TYPE * ) pxISRSpinlock ) + #define data_groupENTER_CRITICAL_FROM_ISR() portENTER_CRITICAL_DATA_GROUP_FROM_ISR( ( portSPINLOCK_TYPE * ) pxISRSpinlock ) + #define data_groupEXIT_CRITICAL() portEXIT_CRITICAL_DATA_GROUP( ( portSPINLOCK_TYPE * ) pxTaskSpinlock, ( portSPINLOCK_TYPE * ) pxISRSpinlock ) + #define data_groupEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) portEXIT_CRITICAL_DATA_GROUP_FROM_ISR( uxSavedInterruptStatus, ( portSPINLOCK_TYPE * ) pxISRSpinlock ) +#else /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + #define data_groupENTER_CRITICAL() taskENTER_CRITICAL() + #define data_groupENTER_CRITICAL_FROM_ISR() taskENTER_CRITICAL_FROM_ISR() + #define data_groupEXIT_CRITICAL() taskEXIT_CRITICAL() + #define data_groupEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ +``` + +In case of the kernel data group (tasks.c), the granular locking macros make use of the existing `vTaskEnter/ExitCritical()` functions to establish critical sections. + + +```c +#if ( portUSING_GRANULAR_LOCKS == 1 ) + #define kernelENTER_CRITICAL() vTaskEnterCritical() + #define kernelENTER_CRITICAL_FROM_ISR() vTaskEnterCriticalFromISR() + #define kernelEXIT_CRITICAL() vTaskExitCritical() + #define kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) vTaskExitCriticalFromISR( uxSavedInterruptStatus ) +#else /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + #define kernelENTER_CRITICAL() taskENTER_CRITICAL() + #define kernelENTER_CRITICAL_FROM_ISR() taskENTER_CRITICAL_FROM_ISR() + #define kernelEXIT_CRITICAL() taskEXIT_CRITICAL() + #define kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) +#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ +``` + +The previous critical section macros, viz., `taskENTER/EXIT_CRITICAL()` are still provided and can be called by users. However, FreeRTOS source code will no longer call them. If called by the users, the port should implement a "user" data group. As a result, if an application previously relied on `taskENTER/EXIT_CRITICAL()` for thread safe access to some user data, the same code is still thread safe with granular locking enabled. + +```c +#define taskENTER_CRITICAL() portENTER_CRITICAL() +#define taskEXIT_CRITICAL() portEXIT_CRITICAL() +#define taskENTER_CRITICAL_FROM_ISR() portENTER_CRITICAL_FROM_ISR() +#define taskEXIT_CRITICAL_FROM_ISR( x ) portEXIT_CRITICAL_FROM_ISR( x ) +``` + +# Porting Interface + +To support **"Dual Spinlock With Data Group Locking"**, ports will need to provide the following macro definitions + +## Port Config + +The ports will need to provide the following port configuration macros + +```c +#define portUSING_GRANULAR_LOCKS 1 // Enables usage of granular locks +#define portCRITICAL_NESTING_IN_TCB 0 // Disable critical nesting in TCB. Ports will need to track their own critical nesting +``` + +## Spinlocks + +Ports will need to provide the following spinlock related macros macros + +```c +/* +Data type for the port's implementation of a spinlock +*/ +#define portSPINLOCK_TYPE port_spinlock_t +``` + +Macros are provided for the spinlocks for initializing them either statically or dynamically. This is reflected in the macros API pattern. + +```c +#define portINIT_SPINLOCK( pxSpinlock ) _port_spinlock_init( pxSpinlock ) +#define portINIT__SPINLOCK_STATIC PORT_SPINLOCK_STATIC_INIT +``` + +## Critical Section Macros + +The port will need to provide implementations for macros to enter/exit a data group critical section according the procedures described above. Typical implementations of each macro is demonstrated below: + +```c +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #define portENTER_CRITICAL_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) vPortEnterCriticalDataGroup( pxTaskSpinlock, pxISRSpinlock ) + #define portENTER_CRITICAL_DATA_GROUP_FROM_ISR( pxISRSpinlock ) uxPortEnterCriticalDataGroupFromISR( pxISRSpinlock ) + #define portEXIT_CRITICAL_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock ) vPortExitCriticalDataGroup( pxTaskSpinlock, pxISRSpinlock ) + #define portEXIT_CRITICAL_DATA_GROUP_FROM_ISR( x, pxISRSpinlock ) vPortExitCriticalDataGroupFromISR( x, pxISRSpinlock ) +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ +``` + +Example implementation of `portENTER_CRITICAL_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock )` and `portEXIT_CRITICAL_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock )`. Note that: + +- `pxTaskSpinlock` is made optional in case users want to create their own data groups that are protected only be a single lock. +- The kernel implements the `xTaskUnlockCanYield()` function to indicate whether an yield should occur when a critical section exits. This function takes into account whether there are any pending yields and whether preemption is currently disabled. + +```c +void vPortEnterCriticalDataGroup( port_spinlock_t *pxTaskSpinlock, port_spinlock_t *pxISRSpinlock ) +{ + portDISABLE_INTERRUPTS(); + + BaseType_t xCoreID = xPortGetCoreID(); + + /* Task spinlock is optional and is always taken first */ + if( pxTaskSpinlock != NULL ) + { + vPortSpinlockTake( pxTaskSpinlock, portMUX_NO_TIMEOUT ); + uxCriticalNesting[ xCoreID ]++; + } + + /* ISR spinlock must always be provided */ + vPortSpinlockTake( pxISRSpinlock, portMUX_NO_TIMEOUT ); + uxCriticalNesting[ xCoreID ]++; +} + +void vPortExitCriticalDataGroup( port_spinlock_t *pxTaskSpinlock, port_spinlock_t *pxISRSpinlock ) +{ + BaseType_t xCoreID = xPortGetCoreID(); + BaseType_t xYieldCurrentTask; + + configASSERT( uxCriticalNesting[ xCoreID ] > 0U ); + + /* Get the xYieldPending stats inside the critical section. */ + xYieldCurrentTask = xTaskUnlockCanYield(); + + /* ISR spinlock must always be provided */ + vPortSpinlockRelease( pxISRSpinlock ); + uxCriticalNesting[ xCoreID ]--; + + /* Task spinlock is optional and is always taken first */ + if( pxTaskSpinlock != NULL ) + { + vPortSpinlockRelease( pxTaskSpinlock); + uxCriticalNesting[ xCoreID ]--; + } + + assert(uxCriticalNesting[ xCoreID ] >= 0); + + if( uxCriticalNesting[ xCoreID ] == 0 ) + { + portENABLE_INTERRUPTS(); + + /* When a task yields in a critical section it just sets xYieldPending to + * true. So now that we have exited the critical section check if xYieldPending + * is true, and if so yield. */ + if( xYieldCurrentTask != pdFALSE ) + { + portYIELD(); + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + } +} +``` + +Example implementation of `portENTER_CRITICAL_DATA_GROUP_FROM_ISR( pxISRSpinlock )` and `portEXIT_CRITICAL_DATA_GROUP_FROM_ISR( x, pxISRSpinlock )`. Note that only `pxISRSpinlock` needs to be provided since ISR critical sections take a single lock. + +```c +UBaseType_t uxPortLockDataGroupFromISR( port_spinlock_t *pxISRSpinlock ) +{ + UBaseType_t uxSavedInterruptStatus = 0; + + uxSavedInterruptStatus = portSET_INTERRUPT_MASK_FROM_ISR(); + + vPortSpinlockTake( pxISRSpinlock, portMUX_NO_TIMEOUT ); + uxCriticalNesting[xPortGetCoreID()]++; + + return uxSavedInterruptStatus; +} + +```c +void vPortUnlockDataGroupFromISR( UBaseType_t uxSavedInterruptStatus, port_spinlock_t *pxISRSpinlock ) +{ + BaseType_t xCoreID = xPortGetCoreID(); + + vPortSpinlockRelease( pxISRSpinlock ); + uxCriticalNesting[ xCoreID ]--; + + assert(uxCriticalNesting[ xCoreID ] >= 0); + + if( uxCriticalNesting[ xCoreID ] == 0 ) + { + portCLEAR_INTERRUPT_MASK_FROM_ISR( uxSavedInterruptStatus ); + } +} +``` + +# Source Specific Changes + +- Added a `xTaskSpinlock` and `xISRSpinlock` to the data structures of each data group +- All calls to `taskENTER/EXIT_CRITICAL[_FROM_ISR]()` have been replaced with `data_groupENTER/EXIT_CRITICAL[_FROM_ISR]()`. +- Added `xTaskUnlockCanYield()` which indicates whether a yield should occur when exiting a critical (i.e., unlocking a data group). Yields should not occur if preemption is disabled (such as when exiting a critical section inside a suspension block). + +## Tasks (Kernel Data Group) + +- Some functions are called from nested critical sections of other data groups, thus an extra critical section call needs to be added to lock/unlock the kernel data group: + - `vTaskInternalSetTimeOutState()` + - `xTaskIncrementTick()` + - `vTaskSwitchContext()` + - `xTaskRemoveFromEventList()` + - `vTaskInternalSetTimeOutState()` + - `eTaskConfirmSleepModeStatus()` + - `xTaskPriorityDisinherit()` + - `pvTaskIncrementMutexHeldCount()` +- Some functions are called from nested suspension blocks of other data gropus, thus an extra suspend/resume call need to be added: + - `vTaskPlaceOnEventList()` + - `vTaskPlaceOnUnorderedEventList()` + - `vTaskPlaceOnEventListRestricted()` +- `prvCheckForRunStateChange()` has been removed +- Updated `vTaskSuspendAll()` and `xTaskResumeAll()` + - Now holds the `xTaskSpinlock` during kernel suspension + - Also increments/decrements `xPreemptionDisable` to prevent yield from occuring when exiting a critical section from inside a kernel suspension block. + +## Queue + +- Added `queueLOCK()` and `queueUNLOCK()` + - If granular locks are disabled, reverts to the previous `prvLockQueue()` and `prvUnlockQueue()` + - If granular locks are enabled, will lock/unlock the queue data group for tasks + +## Event Groups + +- Added `eventLOCK()` and `eventUNLOCK()` + - If granular locks are disabled, reverts to the previous `vTaskSuspendAll()` and `xTaskResumeAll()` calls + - If granular locks are enabled, will lock/unlock the event groups data group for tasks +- `xEventGroupSetBits()` and `vEventGroupDelete()` will manually walk the task lists (which belong to the kernel data group). Thus, an extra `vTaskSuspendAll()`/`xTaskResumeAll()` is added to ensure that the kernel data group is suspended while walking those tasks lists. + +## Stream Buffer + +- Added `sbLOCK()` and `sbUNLOCK()` + - If granular locks are disabled, reverts to the previous `vTaskSuspendAll()` and `xTaskResumeAll()` calls + - If granular locks are enabled, will lock/unlock the stream buffer data group for tasks + +## Timers + +- Timers don't have a lock/unlock function. The existing `vTaskSuspendAll()`/`xTaskResumeAll()` calls are valid as they rely on freezing the tick count which is part of the kernel data group. + +# Prerequisite Refactoring + +A number of refactoring commits have been added to make the addition of granular locking changes simpler: + +1. Move critical sections inside `xTaskPriorityInherit()` + +Currently, `xTaskPriorityInherit()` is called with wrapping critical sections. The critical sections have now be moved inside the function so that they have access to the kernel data group's spinlocks. + +2. Move critical section into `vTaskPriorityDisinheritAfterTimeout()` + +Currently, `vTaskPriorityDisinheritAfterTimeout()` is called wrapping critical sections, where the highest priority waiting task is separately obtained via `prvGetDisinheritPriorityAfterTimeout()`. The critical section and checking of the highest priority have been all been moved into `vTaskPriorityDisinheritAfterTimeout()` as all of these operations access the kernel data group. + +3. Allow `vTaskPreemptionEnable()` to be nested + +Currently, nested calls of `vTaskPreemptionEnable()` is not supported. However, nested calls are required for granular locking due the occurrence of nested suspension across multiple data groups. + +Thus, `vTaskPreemptionEnable()` has been updated to support nested calls. This is done by changing `xPreemptionDisable` to a count, where a non-zero count means that the current task cannot be preempted. + +# TCB Locks + +When `portUSING_GRANULAR_LOCKS == 1` and `configNUMBER_OF_CORES > 1`, each TCB carries an additional spinlock (`xTCBSpinlock`) that guards TCB-specific fields touched by preemption-disable / preemption-enable paths (`uxPreemptionDisable`, `uxDeferredStateChange`). The TCB lock is acquired via `vTaskTCBEnterCritical()` / `vTaskTCBExitCritical()` and lives below the kernel data group in the lock hierarchy. Using a per-TCB lock here avoids contention on the kernel data-group lock for the common case where a task only needs to inspect or update its own preemption-disable state. + +# Deferred State Changes + +Calls that change a task's run state (`vTaskDelete()`, `vTaskSuspend()`) cannot run their full effect while the target task has preemption disabled, because doing so would yield from a TCB whose owner asked not to be preempted. Granular locks introduce `uxDeferredStateChange` on the TCB to record the requested transition (`tskDEFERRED_DELETION`, `tskDEFERRED_SUSPENSION`). The deferred work is then performed at the next `xTaskPreemptionEnableWithYieldStatus()` once the preemption-disable count drops to zero, outside the TCB critical section so that any context switch and the post-state-change ready-list manipulation happen in a regular kernel critical section. + +# ISR-Only Kernel Critical Section Helpers + +`prvKernelEnterISROnlyCritical()` / `prvKernelExitISROnlyCritical()` are internal helpers used to enter the kernel data group from a context that is already inside a surrounding data-group critical section (and therefore already has interrupts disabled and holds the surrounding data group's ISR spinlock). Unlike `kernelENTER_CRITICAL_FROM_ISR()` / `kernelEXIT_CRITICAL_FROM_ISR()`, they do not touch the interrupt state — the outer section owns that — and they only acquire/release the kernel ISR spinlock and adjust the per-core kernel critical-nesting count. `xTaskRemoveFromEventList()` and `xTaskRemoveFromEventListFromISR()` use these helpers so that callers walking an event list while holding e.g. a queue's spinlock do not need to know whether they are in task or ISR context. + +# Event-List TOCTOU Considerations + +Under granular locks, an event list (`xTasksWaitingToSend`, `xTasksWaitingToReceive`, queue-set container waiting list) is protected by the kernel ISR spinlock, while the surrounding queue / event-group critical section only holds that data group's own spinlocks. The kernel API `xTaskRemoveFromEventList()` re-checks list emptiness under the kernel ISR lock before removing an entry, so callers must call it directly rather than gating it on an outer `listLIST_IS_EMPTY()` check. The outer check can race with a concurrent removal from the tick ISR on another core; the inner re-check is authoritative. + +The only places where an outer empty-check is retained on purpose are the `prvUnlockQueue()` drain loops, where the surrounding queue ISR spinlock prevents concurrent additions to the wait list and the outer check exists for early loop termination, not for race avoidance. + +# Known Limitations + +- Under `configRUN_MULTIPLE_PRIORITIES == 1`, the relaxed lock hierarchy means that an unblock triggered by a tick on one core may take effect on the unblocking core a little later than under global locking. The single-highest-priority invariant continues to hold under `configRUN_MULTIPLE_PRIORITIES == 0`. +- The design has been exercised on the Xtensa SMP port (ESP32). Other ports — including the upstream POSIX, ARMv7-M and ARMv8-M SMP ports — need to provide the spinlock, critical-nesting, and yield primitives listed in *Porting Interface* before they can opt in. + +# Performance Metrics + +Performance numbers will be added once the upstream port reference has the porting checklist completed. Initial measurements on ESP32 (Xtensa, dual-core, 240 MHz) show reduced contention on multi-queue workloads where unrelated tasks send to / receive from distinct queues concurrently; full benchmark methodology and numbers will be published with this PR's review pass. +