From 1ae917325c593ee8aabccd30cf5ed72f7c8db8f2 Mon Sep 17 00:00:00 2001 From: Sudeep Mohanty Date: Sat, 6 Jun 2026 11:36:05 +0200 Subject: [PATCH] feat(freertos/smp): Add support for TCB locks --- event_groups.c | 58 ++- include/FreeRTOS.h | 7 +- include/task.h | 57 ++- queue.c | 87 +++- tasks.c | 1166 +++++++++++++++++++++++++++++++++----------- timers.c | 10 + 6 files changed, 1059 insertions(+), 326 deletions(-) diff --git a/event_groups.c b/event_groups.c index c69b96557..ceb3d0d75 100644 --- a/event_groups.c +++ b/event_groups.c @@ -81,6 +81,34 @@ /*-----------------------------------------------------------*/ +/* + * Macros used to lock and unlock an event group. When a task locks an, + * event group, the task will have thread safe non-deterministic access to + * the event group. + * - Concurrent access from other tasks will be blocked by the xTaskSpinlock + * - Concurrent access from ISRs will be pended + * + * When the task unlocks the event group, all pended access attempts are handled. + */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #define event_groupsLOCK( pxEventBits ) taskDATA_GROUP_LOCK( &( ( pxEventBits )->xTaskSpinlock ) ) + #define event_groupsUNLOCK( pxEventBits ) taskDATA_GROUP_UNLOCK( &( ( pxEventBits )->xTaskSpinlock ) ) + #define event_groupsUNLOCK_WITH_YIELD_STATUS( pxEventBits, pxAlreadyYielded ) \ + do { \ + *( pxAlreadyYielded ) = taskDATA_GROUP_UNLOCK( &( ( pxEventBits )->xTaskSpinlock ) ); \ + } while( 0 ) + #else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #define event_groupsLOCK( pxEventBits ) vTaskSuspendAll() + #define event_groupsUNLOCK( pxEventBits ) xTaskResumeAll() + #define event_groupsUNLOCK_WITH_YIELD_STATUS( pxEventBits, pxAlreadyYielded ) \ + do { \ + *( pxAlreadyYielded ) = xTaskResumeAll(); \ + } while( 0 ) + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + +/*-----------------------------------------------------------*/ + + #if ( configSUPPORT_STATIC_ALLOCATION == 1 ) EventGroupHandle_t xEventGroupCreateStatic( StaticEventGroup_t * pxEventGroupBuffer ) @@ -245,7 +273,7 @@ } } } - xAlreadyYielded = xTaskResumeAll(); + event_groupsUNLOCK_WITH_YIELD_STATUS( pxEventBits, &xAlreadyYielded ); if( xTicksToWait != ( TickType_t ) 0 ) { @@ -401,7 +429,7 @@ traceEVENT_GROUP_WAIT_BITS_BLOCK( xEventGroup, uxBitsToWaitFor ); } } - xAlreadyYielded = xTaskResumeAll(); + event_groupsUNLOCK_WITH_YIELD_STATUS( pxEventBits, &xAlreadyYielded ); if( xTicksToWait != ( TickType_t ) 0 ) { @@ -568,6 +596,13 @@ { traceEVENT_GROUP_SET_BITS( xEventGroup, uxBitsToSet ); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + + /* We are about to access the kernel data group non-deterministically, + * thus we suspend the kernel data group.*/ + vTaskSuspendAll(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + pxListItem = listGET_HEAD_ENTRY( pxList ); /* Set the bits. */ @@ -638,8 +673,12 @@ /* Snapshot resulting bits. */ uxReturnBits = pxEventBits->uxEventBits; + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + ( void ) xTaskResumeAll(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ } - ( void ) xTaskResumeAll(); + event_groupsUNLOCK( pxEventBits ); traceRETURN_xEventGroupSetBits( uxReturnBits ); @@ -662,6 +701,13 @@ { traceEVENT_GROUP_DELETE( xEventGroup ); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + + /* We are about to access the kernel data group non-deterministically, + * thus we suspend the kernel data group.*/ + vTaskSuspendAll(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + while( listCURRENT_LIST_LENGTH( pxTasksWaitingForBits ) > ( UBaseType_t ) 0 ) { /* Unblock the task, returning 0 as the event list is being deleted @@ -669,8 +715,12 @@ configASSERT( pxTasksWaitingForBits->xListEnd.pxNext != ( const ListItem_t * ) &( pxTasksWaitingForBits->xListEnd ) ); vTaskRemoveFromUnorderedEventList( pxTasksWaitingForBits->xListEnd.pxNext, eventUNBLOCKED_DUE_TO_BIT_SET ); } + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + ( void ) xTaskResumeAll(); + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ } - ( void ) xTaskResumeAll(); + event_groupsUNLOCK( pxEventBits ); #if ( ( configSUPPORT_DYNAMIC_ALLOCATION == 1 ) && ( configSUPPORT_STATIC_ALLOCATION == 0 ) ) { diff --git a/include/FreeRTOS.h b/include/FreeRTOS.h index e3f3bb300..dc2748d97 100644 --- a/include/FreeRTOS.h +++ b/include/FreeRTOS.h @@ -2942,10 +2942,6 @@ #error configUSE_MUTEXES must be set to 1 to use recursive mutexes #endif -#if ( ( configRUN_MULTIPLE_PRIORITIES == 0 ) && ( configUSE_TASK_PREEMPTION_DISABLE != 0 ) ) - #error configRUN_MULTIPLE_PRIORITIES must be set to 1 to use task preemption disable -#endif - #if ( ( configUSE_PREEMPTION == 0 ) && ( configUSE_TASK_PREEMPTION_DISABLE != 0 ) ) #error configUSE_PREEMPTION must be set to 1 to use task preemption disable #endif @@ -3245,6 +3241,9 @@ typedef struct xSTATIC_TCB #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) BaseType_t xDummy26; #endif + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portSPINLOCK_TYPE xDummy27; + #endif #if ( ( portSTACK_GROWTH > 0 ) || ( configRECORD_STACK_HIGH_ADDRESS == 1 ) ) void * pxDummy8; #endif diff --git a/include/task.h b/include/task.h index 57da38118..5781e17e6 100644 --- a/include/task.h +++ b/include/task.h @@ -296,22 +296,12 @@ typedef enum * \ingroup GranularLocks */ #if ( portUSING_GRANULAR_LOCKS == 1 ) - #define taskDATA_GROUP_ENTER_CRITICAL( pxTaskSpinlock, pxISRSpinlock ) \ - do { \ - /* Disable preemption to avoid task state changes during the critical section. */ \ - vTaskPreemptionDisable( NULL ); \ - { \ - const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); \ - /* Task spinlock is always taken first */ \ - portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ); \ - /* Disable interrupts */ \ - portDISABLE_INTERRUPTS(); \ - /* Take the ISR spinlock next */ \ - portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) ( pxISRSpinlock ) ); \ - /* Increment the critical nesting count */ \ - portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); \ - } \ - } while( 0 ) + +/* Using a function implementation now since the data group entering critical + * section needs to check for run state change. */ + void taskDataGroupEnterCritical( portSPINLOCK_TYPE * pxTaskSpinlock, + portSPINLOCK_TYPE * pxISRSpinlock ); + #define taskDATA_GROUP_ENTER_CRITICAL taskDataGroupEnterCritical #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ /** @@ -365,7 +355,7 @@ typedef enum mtCOVERAGE_TEST_MARKER(); \ } \ /* Re-enable preemption */ \ - prvTaskPreemptionEnable( NULL ); \ + ( void ) xTaskPreemptionEnableWithYieldStatus( NULL ); \ } while( 0 ) #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ @@ -421,12 +411,12 @@ typedef enum * \ingroup GranularLocks */ #if ( portUSING_GRANULAR_LOCKS == 1 ) - #define taskDATA_GROUP_UNLOCK( pxTaskSpinlock ) \ - ( { \ - portRELEASE_SPINLOCK( portGET_CORE_ID(), ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ); \ - /* Re-enable preemption after releasing the task spinlock. */ \ - prvTaskPreemptionEnable( NULL ); \ - } ) + +/* Release the task spinlock and re-enable preemption. + * Returns the yield status reported by xTaskPreemptionEnableWithYieldStatus(). */ + #define taskDATA_GROUP_UNLOCK( pxTaskSpinlock ) \ + ( portRELEASE_SPINLOCK( portGET_CORE_ID(), ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ), \ + xTaskPreemptionEnableWithYieldStatus( NULL ) ) #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ /*----------------------------------------------------------- @@ -1644,7 +1634,7 @@ BaseType_t xTaskResumeFromISR( TaskHandle_t xTaskToResume ) PRIVILEGED_FUNCTION; * switch, otherwise pdFALSE. This is used by the scheduler to determine if a * context switch may be required following the enable. */ - BaseType_t prvTaskPreemptionEnable( const TaskHandle_t xTask ); + BaseType_t xTaskPreemptionEnableWithYieldStatus( const TaskHandle_t xTask ); #endif /*----------------------------------------------------------- @@ -3727,6 +3717,8 @@ void vTaskPlaceOnEventListRestricted( List_t * const pxEventList, * Removes a task from both the specified event list and the list of blocked * tasks, and places it on a ready queue. * + * Do not call this function from an ISR context. Call xTaskRemoveFromEventListFromISR() instead. + * * xTaskRemoveFromEventList()/vTaskRemoveFromUnorderedEventList() will be called * if either an event occurs to unblock a task, or the block timeout period * expires. @@ -3743,6 +3735,23 @@ void vTaskPlaceOnEventListRestricted( List_t * const pxEventList, * making the call, otherwise pdFALSE. */ BaseType_t xTaskRemoveFromEventList( const List_t * const pxEventList ) PRIVILEGED_FUNCTION; + +/* + * THIS FUNCTION MUST NOT BE USED FROM APPLICATION CODE. IT IS ONLY + * INTENDED FOR USE WHEN IMPLEMENTING A PORT OF THE SCHEDULER AND IS + * AN INTERFACE WHICH IS FOR THE EXCLUSIVE USE OF THE SCHEDULER. + * + * THIS FUNCTION MUST BE CALLED WITH INTERRUPTS DISABLED. + * + * Removes a task from both the specified event list and the list of blocked + * tasks, and places it on a ready queue. This function is the ISR-safe version + * of xTaskRemoveFromEventList(). + * + * @return pdTRUE if the task being removed has a higher priority than the task + * making the call, otherwise pdFALSE. + */ +BaseType_t xTaskRemoveFromEventListFromISR( const List_t * const pxEventList ) PRIVILEGED_FUNCTION; + void vTaskRemoveFromUnorderedEventList( ListItem_t * pxEventListItem, const TickType_t xItemValue ) PRIVILEGED_FUNCTION; diff --git a/queue.c b/queue.c index 8df1481e8..f8198348d 100644 --- a/queue.c +++ b/queue.c @@ -2498,7 +2498,15 @@ static void prvUnlockQueue( Queue_t * const pxQueue ) * removed from the queue while the queue was locked. When a queue is * locked items can be added or removed, but the event lists cannot be * updated. */ - taskENTER_CRITICAL(); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + const BaseType_t xCoreID = portGET_CORE_ID(); + + portDISABLE_INTERRUPTS(); + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->xISRSpinlock ) ); + #else + queueENTER_CRITICAL( pxQueue ); + #endif { int8_t cTxLock = pxQueue->cTxLock; @@ -2576,10 +2584,26 @@ static void prvUnlockQueue( Queue_t * const pxQueue ) pxQueue->cTxLock = queueUNLOCKED; } - taskEXIT_CRITICAL(); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->xISRSpinlock ) ); + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + { + portENABLE_INTERRUPTS(); + } + #else + queueEXIT_CRITICAL( pxQueue ); + #endif /* Do the same for the Rx lock. */ - taskENTER_CRITICAL(); + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + portDISABLE_INTERRUPTS(); + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->xISRSpinlock ) ); + #else + queueENTER_CRITICAL( pxQueue ); + #endif { int8_t cRxLock = pxQueue->cRxLock; @@ -3325,8 +3349,63 @@ BaseType_t xQueueIsQueueFullFromISR( const QueueHandle_t xQueue ) /*-----------------------------------------------------------*/ #if ( configUSE_QUEUE_SETS == 1 ) - static BaseType_t prvNotifyQueueSetContainer( const Queue_t * const pxQueue ) + { + BaseType_t xReturn; + + /* Call the generic version with xIsISR = pdFALSE to indicate task context */ + + #if ( portUSING_GRANULAR_LOCKS == 0 ) + { + xReturn = prvNotifyQueueSetContainerGeneric( pxQueue, pdFALSE ); + } + #else + { + const BaseType_t xCoreID = portGET_CORE_ID(); + + /* This API must be called in a critical section which already has preemption + * and interrupt disabled. */ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xTaskSpinlock ) ); + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xISRSpinlock ) ); + { + xReturn = prvNotifyQueueSetContainerGeneric( pxQueue, pdFALSE ); + } + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xISRSpinlock ) ); + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xTaskSpinlock ) ); + } + #endif /* if ( portUSING_GRANULAR_LOCKS == 0 ) */ + + return xReturn; + } + + static BaseType_t prvNotifyQueueSetContainerFromISR( const Queue_t * const pxQueue ) + { + BaseType_t xReturn; + + /* Call the generic version with xIsISR = pdTRUE to indicate ISR context */ + + #if ( portUSING_GRANULAR_LOCKS == 0 ) + { + xReturn = prvNotifyQueueSetContainerGeneric( pxQueue, pdTRUE ); + } + #else + { + const BaseType_t xCoreID = portGET_CORE_ID(); + + /* This API must be called in a critical section which already has interrupt disabled. */ + portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xISRSpinlock ) ); + { + xReturn = prvNotifyQueueSetContainerGeneric( pxQueue, pdTRUE ); + } + portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) &( pxQueue->pxQueueSetContainer->xISRSpinlock ) ); + } + #endif /* if ( portUSING_GRANULAR_LOCKS == 0 ) */ + + return xReturn; + } + + static BaseType_t prvNotifyQueueSetContainerGeneric( const Queue_t * const pxQueue, + const BaseType_t xIsISR ) { Queue_t * pxQueueSetContainer = pxQueue->pxQueueSetContainer; BaseType_t xReturn = pdFALSE; diff --git a/tasks.c b/tasks.c index 8401f2cef..68caa2228 100644 --- a/tasks.c +++ b/tasks.c @@ -354,7 +354,64 @@ * must be valid. This macro is not required in single core since there is only * one core to yield. */ #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - #define prvYieldCore( xCoreID ) \ + #if ( portUSING_GRANULAR_LOCKS == 1 ) + +/* xYieldPending indicates that a yield was requested but could not be + * processed immediately. + * + * This occurs in situations such as: + * 1. Task preemption is disabled. + * 2. A task requests itself to yield while inside a critical section. + * + * In these cases, xYieldPending is set to pdTRUE to indicate that the + * yield request should be serviced after exiting the critical section. + * + * With granular locking, the task disables preemption first before acquiring + * data group locks. A check for run state change is required before acquiring + * the data group locks to preserve priority ordering, for example, in queue + * operations. However, setting xYieldPending alone is not sufficient. Without + * setting the task run state to taskTASK_SCHEDULED_TO_YIELD, the task would + * not be able to relinquish the critical section for higher priority tasks. + * + * Key distinction: + * - xYieldPending: yield is serviced when leaving the critical section. + * - taskTASK_SCHEDULED_TO_YIELD: yield is serviced before entering the + * critical section. */ + #define prvYieldCore( xCoreID ) \ + do { \ + const BaseType_t xCoreToYield = ( xCoreID ); \ + BaseType_t xCurrentCoreID = portGET_CORE_ID(); \ + if( ( xCoreID ) == xCurrentCoreID ) \ + { \ + /* Pending a yield for this core since it is in the critical section. */ \ + xYieldPendings[ xCoreToYield ] = pdTRUE; \ + pxCurrentTCBs[ xCoreToYield ]->xTaskRunState = taskTASK_SCHEDULED_TO_YIELD; \ + } \ + else \ + { \ + portGET_SPINLOCK( xCurrentCoreID, &( pxCurrentTCBs[ xCoreToYield ]->xTCBSpinlock ) ); \ + { \ + if( pxCurrentTCBs[ xCoreToYield ]->uxPreemptionDisable == 0U ) \ + { \ + /* Request other core to yield if it is not requested before. */ \ + if( pxCurrentTCBs[ xCoreToYield ]->xTaskRunState != taskTASK_SCHEDULED_TO_YIELD ) \ + { \ + portYIELD_CORE( xCoreToYield ); \ + pxCurrentTCBs[ xCoreToYield ]->xTaskRunState = taskTASK_SCHEDULED_TO_YIELD; \ + } \ + } \ + else \ + { \ + xYieldPendings[ xCoreToYield ] = pdTRUE; \ + pxCurrentTCBs[ xCoreToYield ]->xTaskRunState = taskTASK_SCHEDULED_TO_YIELD; \ + } \ + } \ + portRELEASE_SPINLOCK( xCurrentCoreID, &( pxCurrentTCBs[ xCoreToYield ]->xTCBSpinlock ) ); \ + } \ + } while( 0 ) + + #else /* if ( portUSING_GRANULAR_LOCKS == 1 ) */ + #define prvYieldCore( xCoreID ) \ do { \ if( ( xCoreID ) == ( BaseType_t ) portGET_CORE_ID() ) \ { \ @@ -378,6 +435,7 @@ } \ } \ } while( 0 ) + #endif /* if ( portUSING_GRANULAR_LOCKS == 1 ) */ #else /* if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ #define prvYieldCore( xCoreID ) \ do { \ @@ -430,6 +488,20 @@ #define kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) taskEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ) #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ +/* + * Helper macros that pick the appropriate critical-section primitive based on + * the number of cores configured. On SMP builds, multi-core access requires + * the kernel data-group lock; on single-core builds, the lighter + * port-provided primitive is sufficient. + */ +#if ( configNUMBER_OF_CORES > 1 ) + #define taskBASE_TYPE_ENTER_CRITICAL() kernelENTER_CRITICAL() + #define taskBASE_TYPE_EXIT_CRITICAL() kernelEXIT_CRITICAL() +#else /* #if ( configNUMBER_OF_CORES > 1 ) */ + #define taskBASE_TYPE_ENTER_CRITICAL() portBASE_TYPE_ENTER_CRITICAL() + #define taskBASE_TYPE_EXIT_CRITICAL() portBASE_TYPE_EXIT_CRITICAL() +#endif /* #if ( configNUMBER_OF_CORES > 1 ) */ + /*-----------------------------------------------------------*/ /* @@ -467,6 +539,10 @@ typedef struct tskTaskControlBlock /* The old naming convention is used to UBaseType_t uxDeferredStateChange; /**< Used to indicate if the task's state change is deferred. */ #endif + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portSPINLOCK_TYPE xTCBSpinlock; + #endif + #if ( ( portSTACK_GROWTH > 0 ) || ( configRECORD_STACK_HIGH_ADDRESS == 1 ) ) StackType_t * pxEndOfStack; /**< Points to the highest valid address for the stack. */ #endif @@ -709,6 +785,12 @@ STATIC portTASK_FUNCTION_PROTO( prvIdleTask, */ STATIC void prvCheckTasksWaitingTermination( void ) PRIVILEGED_FUNCTION; +/* + * Private helper function to remove a task from an event list. This function + * is shared between the task context and ISR context versions. + */ +static BaseType_t prvTaskRemoveFromEventList( const List_t * const pxEventList ) PRIVILEGED_FUNCTION; + /* * The currently executing task is entering the Blocked state. Add the task to * either the current or the overflow delayed task list. @@ -854,10 +936,10 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; #endif /* #if ( configSUPPORT_DYNAMIC_ALLOCATION == 1 ) */ /* - * Helper function to enable preemption for a task. + * Helper function to enable preemption for a task and return yield status. */ #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - BaseType_t prvTaskPreemptionEnable( const TaskHandle_t xTask ) PRIVILEGED_FUNCTION; + BaseType_t xTaskPreemptionEnableWithYieldStatus( const TaskHandle_t xTask ) PRIVILEGED_FUNCTION; #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ /* @@ -901,6 +983,23 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; size_t n ); #endif /* #if ( ( configUSE_TRACE_FACILITY == 1 ) && ( configUSE_STATS_FORMATTING_FUNCTIONS > 0 ) ) */ + +/* + * Helpers used to enter and exit the kernel critical section from a context + * that is already inside a data-group critical section (and therefore already + * holds the data-group's ISR spinlock with interrupts disabled). + * + * Unlike kernelENTER_CRITICAL_FROM_ISR/kernelEXIT_CRITICAL_FROM_ISR, these + * helpers do not touch the interrupt state: the outer data-group critical + * section is responsible for that. They only acquire/release the kernel + * ISR spinlock and adjust the per-core kernel critical-nesting count, so + * that callers can safely access kernel-data-group members while remaining + * within the surrounding data-group critical section's lock hierarchy. + */ +static void prvKernelEnterISROnlyCritical( void ); + +static void prvKernelExitISROnlyCritical( void ); + /*-----------------------------------------------------------*/ #if ( configNUMBER_OF_CORES > 1 ) @@ -929,20 +1028,11 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; * and reacquire the correct locks. And then, do it all over again * if our state changed again during the reacquisition. */ uxPrevCriticalNesting = portGET_CRITICAL_NESTING_COUNT( xCoreID ); + portSET_CRITICAL_NESTING_COUNT( xCoreID, 0U ); - if( uxPrevCriticalNesting > 0U ) - { - portSET_CRITICAL_NESTING_COUNT( xCoreID, 0U ); - kernelRELEASE_ISR_LOCK( xCoreID ); - } - else - { - /* The scheduler is suspended. uxSchedulerSuspended is updated - * only when the task is not requested to yield. */ - mtCOVERAGE_TEST_MARKER(); - } - + kernelRELEASE_ISR_LOCK( xCoreID ); kernelRELEASE_TASK_LOCK( xCoreID ); + portMEMORY_BARRIER(); configASSERT( pxThisTCB->xTaskRunState == taskTASK_SCHEDULED_TO_YIELD ); @@ -957,15 +1047,11 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; portDISABLE_INTERRUPTS(); xCoreID = ( BaseType_t ) portGET_CORE_ID(); + kernelGET_TASK_LOCK( xCoreID ); kernelGET_ISR_LOCK( xCoreID ); portSET_CRITICAL_NESTING_COUNT( xCoreID, uxPrevCriticalNesting ); - - if( uxPrevCriticalNesting == 0U ) - { - kernelRELEASE_ISR_LOCK( xCoreID ); - } } } #endif /* #if ( configNUMBER_OF_CORES > 1 ) */ @@ -1028,14 +1114,30 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; { #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) { + /* Acquire the preemption lock to prevent the task changes + * it's preemption state. */ + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portGET_SPINLOCK( xCurrentCoreID, &( pxCurrentTCBs[ xCoreID ]->xTCBSpinlock ) ); + #endif + if( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) { + #if ( portUSING_GRANULAR_LOCKS == 1 ) + if( xLowestPriorityCore != -1 ) + { + /* Release previous preemption lock. */ + portRELEASE_SPINLOCK( xCurrentCoreID, &( pxCurrentTCBs[ xLowestPriorityCore ]->xTCBSpinlock ) ); + } + #endif xLowestPriorityToPreempt = xCurrentCoreTaskPriority; xLowestPriorityCore = xCoreID; } else { xYieldPendings[ xCoreID ] = pdTRUE; + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portRELEASE_SPINLOCK( xCurrentCoreID, &( pxCurrentTCBs[ xCoreID ]->xTCBSpinlock ) ); + #endif } } #else /* if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ @@ -1084,6 +1186,18 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; prvYieldCore( xLowestPriorityCore ); } + /* After request the core to yield, preemption lock can be released. + * The task tries to set the preemption disable count will check it's + * run state first. If it is already requested to yield, it will serve + * the yield request first then retring to set the preemption disable + * count. */ + #if ( portUSING_GRANULAR_LOCKS == 1 ) + if( xLowestPriorityCore >= 0 ) + { + portRELEASE_SPINLOCK( xCurrentCoreID, &( pxCurrentTCBs[ xLowestPriorityCore ]->xTCBSpinlock ) ); + } + #endif + #if ( configRUN_MULTIPLE_PRIORITIES == 0 ) /* Verify that the calling core always yields to higher priority tasks. */ if( ( ( pxCurrentTCBs[ xCurrentCoreID ]->uxTaskAttributes & taskATTRIBUTE_IS_IDLE ) == 0U ) && @@ -1264,9 +1378,13 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; for( x = ( BaseType_t ) 0; x < ( BaseType_t ) configNUMBER_OF_CORES; x++ ) { - if( ( pxCurrentTCBs[ x ]->uxTaskAttributes & taskATTRIBUTE_IS_IDLE ) != 0U ) + /* Core xCoreID already selects the task to run. */ + if( x != xCoreID ) { - prvYieldCore( x ); + if( ( pxCurrentTCBs[ x ]->uxTaskAttributes & taskATTRIBUTE_IS_IDLE ) != 0U ) + { + prvYieldCore( x ); + } } } } @@ -1344,17 +1462,28 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; ( xYieldPendings[ uxCore ] == pdFALSE ) ) { #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - { + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portGET_SPINLOCK( xCoreID, &( pxCurrentTCBs[ uxCore ]->xTCBSpinlock ) ); + #endif + if( pxCurrentTCBs[ uxCore ]->uxPreemptionDisable == 0U ) { + #if ( portUSING_GRANULAR_LOCKS == 1 ) + if( xLowestPriorityCore != -1 ) + { + portRELEASE_SPINLOCK( xCoreID, &( pxCurrentTCBs[ xLowestPriorityCore ]->xTCBSpinlock ) ); + } + #endif xLowestPriority = xTaskPriority; xLowestPriorityCore = ( BaseType_t ) uxCore; } else { xYieldPendings[ uxCore ] = pdTRUE; + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portRELEASE_SPINLOCK( xCoreID, &( pxCurrentTCBs[ uxCore ]->xTCBSpinlock ) ); + #endif } - } #else /* if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ { xLowestPriority = xTaskPriority; @@ -1368,6 +1497,10 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION; if( xLowestPriorityCore >= 0 ) { prvYieldCore( xLowestPriorityCore ); + + #if ( portUSING_GRANULAR_LOCKS == 1 ) + portRELEASE_SPINLOCK( xCoreID, &( pxCurrentTCBs[ xLowestPriorityCore ]->xTCBSpinlock ) ); + #endif } } } @@ -2066,6 +2199,12 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, } #endif + #if ( portUSING_GRANULAR_LOCKS == 1 ) + { + portINIT_SPINLOCK( &pxNewTCB->xTCBSpinlock ); + } + #endif + /* Initialize the TCB stack to look as if the task was already running, * but had been interrupted by the scheduler. The return address is set * to the start of the task function. Once the stack has been initialised @@ -2342,25 +2481,45 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, kernelENTER_CRITICAL(); { + #if ( ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) ) + const BaseType_t xCoreID = portGET_CORE_ID(); + #endif /* ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) */ + /* If null is passed in here then it is the calling task that is * being deleted. */ pxTCB = prvGetTCBFromHandle( xTaskToDelete ); configASSERT( pxTCB != NULL ); - #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - - /* If the task has disabled preemption, we need to defer the deletion until the - * task enables preemption. The deletion will be performed in vTaskPreemptionEnable(). */ - if( pxTCB->uxPreemptionDisable > 0U ) + #if ( ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) ) + { + /* Acquire the TCB lock before reading the preemption-disable + * count to avoid a race with the target task running on another + * core (which may be modifying uxPreemptionDisable concurrently). */ + portGET_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); { - pxTCB->uxDeferredStateChange |= tskDEFERRED_DELETION; - xDeferredDeletion = pdTRUE; + /* If the task has disabled preemption, we need to defer the deletion until the + * task enables preemption. The deletion will be performed in vTaskPreemptionEnable(). */ + if( pxTCB->uxPreemptionDisable > 0U ) + { + /* Deletion takes priority over any other deferred action. + * Overwrite the flag word so that a previously-queued + * tskDEFERRED_SUSPENSION cannot be dispatched after this point. + * tskDEFERRED_DELETION will be dispatched in vTaskPreemptionEnable(). */ + pxTCB->uxDeferredStateChange = tskDEFERRED_DELETION; + xDeferredDeletion = pdTRUE; + portRELEASE_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); + } + else + { + /* Reset all deferred state change flags since the task is + * being deleted now. Any pending tskDEFERRED_SUSPENSION is irrelevant. + * Failing to clear it would leave the tskDEFERRED_SUSPENSION latched + * after the deletion completes, preventing the calling context from yielding. */ + pxTCB->uxDeferredStateChange = 0U; + } } - else - { - mtCOVERAGE_TEST_MARKER(); - } - #endif /* configUSE_TASK_PREEMPTION_DISABLE */ + } + #endif /* ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) */ #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) if( xDeferredDeletion == pdFALSE ) @@ -2466,6 +2625,10 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, * the task that has just been deleted. */ prvResetNextTaskUnblockTime(); } + + #if ( ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) ) + portRELEASE_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); + #endif } } kernelEXIT_CRITICAL(); @@ -2791,15 +2954,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, traceENTER_uxTaskPriorityGet( xTask ); - #if ( ( configNUMBER_OF_CORES > 1 ) ) - { - kernelENTER_CRITICAL(); - } - #else - { - portBASE_TYPE_ENTER_CRITICAL(); - } - #endif + taskBASE_TYPE_ENTER_CRITICAL(); { /* If null is passed in here then it is the priority of the task * that called uxTaskPriorityGet() that is being queried. */ @@ -2808,15 +2963,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, uxReturn = pxTCB->uxPriority; } - #if ( ( configNUMBER_OF_CORES > 1 ) ) - { - kernelEXIT_CRITICAL(); - } - #else - { - portBASE_TYPE_EXIT_CRITICAL(); - } - #endif + taskBASE_TYPE_EXIT_CRITICAL(); traceRETURN_uxTaskPriorityGet( uxReturn ); @@ -2885,15 +3032,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, traceENTER_uxTaskBasePriorityGet( xTask ); - #if ( ( configNUMBER_OF_CORES > 1 ) ) - { - kernelENTER_CRITICAL(); - } - #else - { - portBASE_TYPE_ENTER_CRITICAL(); - } - #endif + taskBASE_TYPE_ENTER_CRITICAL(); { /* If null is passed in here then it is the base priority of the task * that called uxTaskBasePriorityGet() that is being queried. */ @@ -2902,15 +3041,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, uxReturn = pxTCB->uxBasePriority; } - #if ( ( configNUMBER_OF_CORES > 1 ) ) - { - kernelEXIT_CRITICAL(); - } - #else - { - portBASE_TYPE_EXIT_CRITICAL(); - } - #endif + taskBASE_TYPE_EXIT_CRITICAL(); traceRETURN_uxTaskBasePriorityGet( uxReturn ); @@ -3245,30 +3376,14 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, traceENTER_vTaskCoreAffinityGet( xTask ); - #if ( ( configNUMBER_OF_CORES > 1 ) ) - { - kernelENTER_CRITICAL(); - } - #else - { - portBASE_TYPE_ENTER_CRITICAL(); - } - #endif + taskBASE_TYPE_ENTER_CRITICAL(); { pxTCB = prvGetTCBFromHandle( xTask ); configASSERT( pxTCB != NULL ); uxCoreAffinityMask = pxTCB->uxCoreAffinityMask; } - #if ( ( configNUMBER_OF_CORES > 1 ) ) - { - kernelEXIT_CRITICAL(); - } - #else - { - portBASE_TYPE_EXIT_CRITICAL(); - } - #endif + taskBASE_TYPE_EXIT_CRITICAL(); traceRETURN_vTaskCoreAffinityGet( uxCoreAffinityMask ); @@ -3278,15 +3393,232 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, /*-----------------------------------------------------------*/ -#if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) +#if ( portUSING_GRANULAR_LOCKS == 1 ) + static void prvTaskTCBLockCheckForRunStateChange( void ) + { + const TCB_t * pxThisTCB; + BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + + /* This must only be called from within a task. */ + portASSERT_IF_IN_ISR(); + + /* This function is always called with interrupts disabled + * so this is safe. */ + pxThisTCB = pxCurrentTCBs[ xCoreID ]; + + while( pxThisTCB->xTaskRunState == taskTASK_SCHEDULED_TO_YIELD ) + { + UBaseType_t uxPrevCriticalNesting; + + /* We are only here if we just entered a critical section + * or if we just suspended the scheduler, and another task + * has requested that we yield. + * + * This is slightly complicated since we need to save and restore + * the suspension and critical nesting counts, as well as release + * and reacquire the correct locks. And then, do it all over again + * if our state changed again during the reacquisition. */ + uxPrevCriticalNesting = portGET_CRITICAL_NESTING_COUNT( xCoreID ); + + if( uxPrevCriticalNesting > 0U ) + { + portSET_CRITICAL_NESTING_COUNT( xCoreID, 0U ); + portRELEASE_SPINLOCK( xCoreID, &pxCurrentTCBs[ xCoreID ]->xTCBSpinlock ); + } + else + { + /* The scheduler is suspended. uxSchedulerSuspended is updated + * only when the task is not requested to yield. */ + mtCOVERAGE_TEST_MARKER(); + } + + portMEMORY_BARRIER(); + + portENABLE_INTERRUPTS(); + + /* Enabling interrupts should cause this core to immediately service + * the pending interrupt and yield. After servicing the pending interrupt, + * the task needs to re-evaluate its run state within this loop, as + * other cores may have requested this task to yield, potentially altering + * its run state. */ + + portDISABLE_INTERRUPTS(); + + xCoreID = ( BaseType_t ) portGET_CORE_ID(); + portGET_SPINLOCK( xCoreID, &pxCurrentTCBs[ xCoreID ]->xTCBSpinlock ); + + portSET_CRITICAL_NESTING_COUNT( xCoreID, uxPrevCriticalNesting ); + } + } + + void vTaskTCBEnterCritical( void ) + { + if( xSchedulerRunning != pdFALSE ) + { + portDISABLE_INTERRUPTS(); + { + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + + portGET_SPINLOCK( xCoreID, &pxCurrentTCBs[ xCoreID ]->xTCBSpinlock ); + + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + + if( ( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 1U ) && + ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) && + ( uxSchedulerSuspended == 0U ) ) + { + prvTaskTCBLockCheckForRunStateChange(); + } + } + } + } + + void vTaskTCBExitCritical( void ) + { + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + + if( xSchedulerRunning != pdFALSE ) + { + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ) + { + BaseType_t xYieldCurrentTask = pdFALSE; + + /* Get the xYieldPending stats inside the critical section. */ + if( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) + { + xYieldCurrentTask = xYieldPendings[ xCoreID ]; + } + + portRELEASE_SPINLOCK( xCoreID, &pxCurrentTCBs[ xCoreID ]->xTCBSpinlock ); + + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + + /* If the critical nesting count is 0, enable interrupts */ + if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + { + portENABLE_INTERRUPTS(); + + if( xYieldCurrentTask != pdFALSE ) + { + portYIELD(); + } + } + } + } + } + + static void prvTaskDataGroupCheckForRunStateChange( portSPINLOCK_TYPE * pxTaskSpinlock, + portSPINLOCK_TYPE * pxISRSpinlock ) + { + const TCB_t * pxThisTCB; + BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + + /* This must only be called from within a task. */ + portASSERT_IF_IN_ISR(); + + /* This function is always called with interrupts disabled + * so this is safe. */ + pxThisTCB = pxCurrentTCBs[ xCoreID ]; + + while( pxThisTCB->xTaskRunState == taskTASK_SCHEDULED_TO_YIELD ) + { + UBaseType_t uxPrevCriticalNesting; + + /* We are only here if we just entered a critical section + * or if we just suspended the scheduler, and another task + * has requested that we yield. + * + * This is slightly complicated since we need to save and restore + * the suspension and critical nesting counts, as well as release + * and reacquire the correct locks. And then, do it all over again + * if our state changed again during the reacquisition. */ + uxPrevCriticalNesting = portGET_CRITICAL_NESTING_COUNT( xCoreID ); + + /* Data group lock. */ + if( uxPrevCriticalNesting > 0U ) + { + portSET_CRITICAL_NESTING_COUNT( xCoreID, 0U ); + portRELEASE_SPINLOCK( xCoreID, pxISRSpinlock ); + } + else + { + /* The scheduler is suspended. uxSchedulerSuspended is updated + * only when the task is not requested to yield. */ + mtCOVERAGE_TEST_MARKER(); + } + + portENABLE_INTERRUPTS(); + + portRELEASE_SPINLOCK( xCoreID, pxTaskSpinlock ); + + portMEMORY_BARRIER(); + + xTaskPreemptionEnableWithYieldStatus( NULL ); + + + + /* Enabling interrupts should cause this core to immediately service + * the pending interrupt and yield. After servicing the pending interrupt, + * the task needs to re-evaluate its run state within this loop, as + * other cores may have requested this task to yield, potentially altering + * its run state. */ + + vTaskPreemptionDisable( NULL ); + + xCoreID = ( BaseType_t ) portGET_CORE_ID(); + portGET_SPINLOCK( xCoreID, pxTaskSpinlock ); + /* Disable interrupts */ + portDISABLE_INTERRUPTS(); + /* Take the ISR spinlock next */ + portGET_SPINLOCK( xCoreID, pxISRSpinlock ); + /* Increment the critical nesting count */ + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + } + } + + void taskDataGroupEnterCritical( portSPINLOCK_TYPE * pxTaskSpinlock, + portSPINLOCK_TYPE * pxISRSpinlock ) + { + /* Disable preemption to avoid task state changes during the critical section. */ + vTaskPreemptionDisable( NULL ); + { + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + /* Task spinlock is always taken first */ + portGET_SPINLOCK( xCoreID, pxTaskSpinlock ); + /* Disable interrupts */ + portDISABLE_INTERRUPTS(); + /* Take the ISR spinlock next */ + portGET_SPINLOCK( xCoreID, pxISRSpinlock ); + /* Increment the critical nesting count */ + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + + if( xSchedulerRunning == pdTRUE ) + { + if( ( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 1U ) && + ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 1U ) && + ( uxSchedulerSuspended == 0U ) ) + { + prvTaskDataGroupCheckForRunStateChange( pxTaskSpinlock, pxISRSpinlock ); + } + } + } + } +#endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ +/*-----------------------------------------------------------*/ + +#if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) void vTaskPreemptionDisable( const TaskHandle_t xTask ) { TCB_t * pxTCB; traceENTER_vTaskPreemptionDisable( xTask ); - kernelENTER_CRITICAL(); + #if ( portUSING_GRANULAR_LOCKS == 1 ) + vTaskTCBEnterCritical(); + #else + kernelENTER_CRITICAL(); + #endif { if( xSchedulerRunning != pdFALSE ) { @@ -3300,7 +3632,11 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, mtCOVERAGE_TEST_MARKER(); } } - kernelEXIT_CRITICAL(); + #if ( portUSING_GRANULAR_LOCKS == 1 ) + vTaskTCBExitCritical(); + #else + kernelEXIT_CRITICAL(); + #endif traceRETURN_vTaskPreemptionDisable(); } @@ -3310,18 +3646,23 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - BaseType_t prvTaskPreemptionEnable( const TaskHandle_t xTask ) + BaseType_t xTaskPreemptionEnableWithYieldStatus( const TaskHandle_t xTask ) { TCB_t * pxTCB; - UBaseType_t uxDeferredAction = 0U; + BaseType_t xHasDeferredAction = pdFALSE; BaseType_t xAlreadyYielded = pdFALSE; + BaseType_t xTaskRequestedToYield = pdFALSE; - traceENTER_vTaskPreemptionEnable( xTask ); - - kernelENTER_CRITICAL(); + #if ( portUSING_GRANULAR_LOCKS == 1 ) + vTaskTCBEnterCritical(); + #else + kernelENTER_CRITICAL(); + #endif { if( xSchedulerRunning != pdFALSE ) { + /* Current task running on the core can not be changed by other core. + * Get TCB from handle is safe to call within TCB critical section. */ pxTCB = prvGetTCBFromHandle( xTask ); configASSERT( pxTCB != NULL ); configASSERT( pxTCB->uxPreemptionDisable > 0U ); @@ -3332,14 +3673,20 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, { if( pxTCB->uxDeferredStateChange != 0U ) { - uxDeferredAction = pxTCB->uxDeferredStateChange; + /* A deferred state change is pending. Record only the fact + * that a deferred action is pending, rather than the specific + * deferred action itself. This allows us to tolerate a concurrent + * vTaskDelete()/vTaskSuspend() on another core that may have + * upgraded the deferred action between the snapshot and the dispatch. + * The actual deferred action will be read and dispatched in the + * loop below after exiting the TCB critical section. */ + xHasDeferredAction = pdTRUE; } else { if( ( xYieldPendings[ pxTCB->xTaskRunState ] != pdFALSE ) && ( taskTASK_IS_RUNNING( pxTCB ) != pdFALSE ) ) { - prvYieldCore( pxTCB->xTaskRunState ); - xAlreadyYielded = pdTRUE; + xTaskRequestedToYield = pdTRUE; } else { @@ -3357,25 +3704,53 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, mtCOVERAGE_TEST_MARKER(); } } - kernelEXIT_CRITICAL(); + #if ( portUSING_GRANULAR_LOCKS == 1 ) + vTaskTCBExitCritical(); + #else + kernelEXIT_CRITICAL(); + #endif - if( uxDeferredAction != 0U ) + if( xHasDeferredAction != pdFALSE ) { - if( uxDeferredAction & tskDEFERRED_DELETION ) + /* Read uxDeferredStateChange after exiting the TCB critical section. + * Between the snapshot and here, another core may have upgraded the + * deferred action. */ + if( pxTCB->uxDeferredStateChange & tskDEFERRED_DELETION ) { vTaskDelete( xTask ); + xAlreadyYielded = pdTRUE; } - else if( uxDeferredAction & tskDEFERRED_SUSPENSION ) + else if( pxTCB->uxDeferredStateChange & tskDEFERRED_SUSPENSION ) { vTaskSuspend( xTask ); + xAlreadyYielded = pdTRUE; } else { + /* Concurrent operation cleared the deferred state — nothing to do. */ mtCOVERAGE_TEST_MARKER(); } + } + else + { + if( xTaskRequestedToYield != pdFALSE ) + { + /* prvYieldCore must be called in critical section. */ + kernelENTER_CRITICAL(); + { + pxTCB = prvGetTCBFromHandle( xTask ); - /* Any deferred action on the task would result in a context switch. */ - xAlreadyYielded = pdTRUE; + /* There is gap between TCB critical section and kernel critical section. + * Checking the yield pending again to prevent that the current task + * already handle the yield request. */ + if( ( xYieldPendings[ pxTCB->xTaskRunState ] != pdFALSE ) && ( taskTASK_IS_RUNNING( pxTCB ) != pdFALSE ) ) + { + prvYieldCore( pxTCB->xTaskRunState ); + } + } + kernelEXIT_CRITICAL(); + xAlreadyYielded = pdTRUE; + } } return xAlreadyYielded; @@ -3389,7 +3764,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, { traceENTER_vTaskPreemptionEnable( xTask ); - ( void ) prvTaskPreemptionEnable( xTask ); + ( void ) xTaskPreemptionEnableWithYieldStatus( xTask ); traceRETURN_vTaskPreemptionEnable(); } @@ -3411,30 +3786,72 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, kernelENTER_CRITICAL(); { + #if ( ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) ) + const BaseType_t xCoreID = portGET_CORE_ID(); + #endif /* ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) */ + /* If null is passed in here then it is the running task that is * being suspended. */ pxTCB = prvGetTCBFromHandle( xTaskToSuspend ); configASSERT( pxTCB != NULL ); - #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + #if ( ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) ) + { + /* Acquire the TCB lock before reading the preemption-disable + * count to avoid a race with the target task running on another + * core (which may be modifying uxPreemptionDisable concurrently). */ + portGET_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); /* If the task has disabled preemption, we need to defer the suspension until the * task enables preemption. The suspension will be performed in vTaskPreemptionEnable(). */ if( pxTCB->uxPreemptionDisable > 0U ) { - pxTCB->uxDeferredStateChange |= tskDEFERRED_SUSPENSION; + /* Only queue the suspension if a deletion is not already deferred. + * Deletion takes priority — if both are requested, the dispatcher + * will only process the deletion. Recording tskDEFERRED_SUSPENSION on top of + * a pending tskDEFERRED_DELETION would have no effect. */ + if( ( pxTCB->uxDeferredStateChange & tskDEFERRED_DELETION ) == 0U ) + { + pxTCB->uxDeferredStateChange |= tskDEFERRED_SUSPENSION; + } + xDeferredSuspension = pdTRUE; + portRELEASE_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); } else { - mtCOVERAGE_TEST_MARKER(); + /* Reset the deferred state change flags */ + pxTCB->uxDeferredStateChange &= ~tskDEFERRED_SUSPENSION; } - #endif /* configUSE_TASK_PREEMPTION_DISABLE */ + } + #endif /* ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) */ #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) if( xDeferredSuspension == pdFALSE ) #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ { + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) && ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) ) + { + /* In SMP + granular-locks builds, an earlier-deferred suspension + * can be dispatched at the same time another core is calling + * vTaskDelete() on the same TCB. The dispatcher in + * xTaskPreemptionEnableWithYieldStatus releases the TCB + * spinlock before calling us, leaving a small window in which + * vTaskDelete can complete its immediate path and move the TCB + * onto xTasksWaitingTermination. If that has happened, the + * deletion takes precedence — abort here rather than moving + * the TCB off the termination list (which would silently drop + * the deletion request and leak the TCB). */ + if( listLIST_ITEM_CONTAINER( &( pxTCB->xStateListItem ) ) == &xTasksWaitingTermination ) + { + portRELEASE_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); + kernelEXIT_CRITICAL(); + traceRETURN_vTaskSuspend(); + return; + } + } + #endif /* ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) && ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + traceTASK_SUSPEND( pxTCB ); /* Remove task from the ready/delayed list and place in the @@ -3515,6 +3932,9 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, } } #endif /* #if ( configNUMBER_OF_CORES > 1 ) */ + #if ( ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( portUSING_GRANULAR_LOCKS == 1 ) ) + portRELEASE_SPINLOCK( xCoreID, &pxTCB->xTCBSpinlock ); + #endif } } kernelEXIT_CRITICAL(); @@ -3692,7 +4112,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, kernelENTER_CRITICAL(); { #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - + { /* If the task being resumed is in a deferred suspension state, * we simply clear the deferred suspension state and return. */ if( pxTCB->uxDeferredStateChange & tskDEFERRED_SUSPENSION ) @@ -3704,6 +4124,7 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, { mtCOVERAGE_TEST_MARKER(); } + } #endif /* configUSE_TASK_PREEMPTION_DISABLE */ #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) @@ -3752,6 +4173,10 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, TCB_t * const pxTCB = xTaskToResume; UBaseType_t uxSavedInterruptStatus; + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + BaseType_t xTaskResumed = pdFALSE; + #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + traceENTER_xTaskResumeFromISR( xTaskToResume ); configASSERT( xTaskToResume ); @@ -3779,58 +4204,82 @@ STATIC void prvInitialiseNewTask( TaskFunction_t pxTaskCode, /* coverity[misra_c_2012_directive_4_7_violation] */ uxSavedInterruptStatus = kernelENTER_CRITICAL_FROM_ISR(); { - if( prvTaskIsTaskSuspended( pxTCB ) != pdFALSE ) + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) { - traceTASK_RESUME_FROM_ISR( pxTCB ); + /* FIXME : Acquire the TCB lock before reading the preemptions + * disable count. */ - /* Check the ready lists can be accessed. */ - if( uxSchedulerSuspended == ( UBaseType_t ) 0U ) + /* If the task being resumed is in a deferred suspension state, + * we simply clear the deferred suspension state and return. */ + if( pxTCB->uxDeferredStateChange & tskDEFERRED_SUSPENSION ) { - #if ( configNUMBER_OF_CORES == 1 ) - { - /* Ready lists can be accessed so move the task from the - * suspended list to the ready list directly. */ - if( pxTCB->uxPriority > pxCurrentTCB->uxPriority ) - { - xYieldRequired = pdTRUE; - - /* Mark that a yield is pending in case the user is not - * using the return value to initiate a context switch - * from the ISR using the port specific portYIELD_FROM_ISR(). */ - xYieldPendings[ 0 ] = pdTRUE; - } - else - { - mtCOVERAGE_TEST_MARKER(); - } - } - #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ - - ( void ) uxListRemove( &( pxTCB->xStateListItem ) ); - prvAddTaskToReadyList( pxTCB ); + pxTCB->uxDeferredStateChange &= ~tskDEFERRED_SUSPENSION; + xTaskResumed = pdTRUE; } else { - /* The delayed or ready lists cannot be accessed so the task - * is held in the pending ready list until the scheduler is - * unsuspended. */ - vListInsertEnd( &( xPendingReadyList ), &( pxTCB->xEventListItem ) ); + mtCOVERAGE_TEST_MARKER(); } - - #if ( ( configNUMBER_OF_CORES > 1 ) && ( configUSE_PREEMPTION == 1 ) ) - { - prvYieldForTask( pxTCB ); - - if( xYieldPendings[ portGET_CORE_ID() ] != pdFALSE ) - { - xYieldRequired = pdTRUE; - } - } - #endif /* #if ( ( configNUMBER_OF_CORES > 1 ) && ( configUSE_PREEMPTION == 1 ) ) */ } - else + #endif /* configUSE_TASK_PREEMPTION_DISABLE */ + + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + if( xTaskResumed == pdFALSE ) + #endif /* configUSE_TASK_PREEMPTION_DISABLE */ { - mtCOVERAGE_TEST_MARKER(); + if( prvTaskIsTaskSuspended( pxTCB ) != pdFALSE ) + { + traceTASK_RESUME_FROM_ISR( pxTCB ); + + /* Check the ready lists can be accessed. */ + if( uxSchedulerSuspended == ( UBaseType_t ) 0U ) + { + #if ( configNUMBER_OF_CORES == 1 ) + { + /* Ready lists can be accessed so move the task from the + * suspended list to the ready list directly. */ + if( pxTCB->uxPriority > pxCurrentTCB->uxPriority ) + { + xYieldRequired = pdTRUE; + + /* Mark that a yield is pending in case the user is not + * using the return value to initiate a context switch + * from the ISR using the port specific portYIELD_FROM_ISR(). */ + xYieldPendings[ 0 ] = pdTRUE; + } + else + { + mtCOVERAGE_TEST_MARKER(); + } + } + #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ + + ( void ) uxListRemove( &( pxTCB->xStateListItem ) ); + prvAddTaskToReadyList( pxTCB ); + } + else + { + /* The delayed or ready lists cannot be accessed so the task + * is held in the pending ready list until the scheduler is + * unsuspended. */ + vListInsertEnd( &( xPendingReadyList ), &( pxTCB->xEventListItem ) ); + } + + #if ( ( configNUMBER_OF_CORES > 1 ) && ( configUSE_PREEMPTION == 1 ) ) + { + prvYieldForTask( pxTCB ); + + if( xYieldPendings[ portGET_CORE_ID() ] != pdFALSE ) + { + xYieldRequired = pdTRUE; + } + } + #endif /* #if ( ( configNUMBER_OF_CORES > 1 ) && ( configUSE_PREEMPTION == 1 ) ) */ + } + else + { + mtCOVERAGE_TEST_MARKER(); + } } } kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); @@ -4217,6 +4666,13 @@ void vTaskSuspendAll( void ) kernelGET_TASK_LOCK( xCoreID ); + /* Task lock is acquired here to increase the scheduler suspended count. + * In granular lock, it is possible that prvYieldCore is called with only + * ISR spinlock is acquired in ISR. To remove the gap, the logic is updated + * to acquire the ISR spinlock before check for run state. The logic in + * prvCheckForRunStateChange() is updated accordingly. */ + kernelGET_ISR_LOCK( xCoreID ); + /* uxSchedulerSuspended is increased after prvCheckForRunStateChange. The * purpose is to prevent altering the variable when fromISR APIs are readying * it. */ @@ -4225,9 +4681,14 @@ void vTaskSuspendAll( void ) /* If the scheduler is being suspended after taking a granular lock (in any data group) * then we do not check for the run state of a task and we let it run through until * the granular lock is released. */ + + /* Task may acquire kernel lock in data group critical section. + * In this case, check for run state change is not required since + * we can't yield for other task when preemption is disabled. */ #if ( portUSING_GRANULAR_LOCKS == 1 ) && ( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */ + && ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) ) { prvCheckForRunStateChange(); @@ -4242,8 +4703,6 @@ void vTaskSuspendAll( void ) * task lock for the core is acquired in prvCheckForRunStateChange. */ xCoreID = ( BaseType_t ) portGET_CORE_ID(); - kernelGET_ISR_LOCK( xCoreID ); - /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment * is used to allow calls to vTaskSuspendAll() to nest. */ ++uxSchedulerSuspended; @@ -4388,9 +4847,9 @@ BaseType_t xTaskResumeAll( void ) } #else /* #if ( configNUMBER_OF_CORES == 1 ) */ { - /* All appropriate tasks yield at the moment a task is added to xPendingReadyList. - * If the current core yielded then vTaskSwitchContext() has already been called - * which sets xYieldPendings for the current core to pdTRUE. */ + /* Yield for the task when removing out from pending + * ready list. */ + prvYieldForTask( pxTCB ); } #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ } @@ -5053,6 +5512,10 @@ BaseType_t xTaskIncrementTick( void ) UBaseType_t uxSavedInterruptStatus; #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configUSE_TICK_HOOK == 1 ) ) + BaseType_t xApplicationTickRequired = pdFALSE; + #endif + traceENTER_xTaskIncrementTick(); #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) @@ -5227,7 +5690,15 @@ BaseType_t xTaskIncrementTick( void ) * count is being unwound (when the scheduler is being unlocked). */ if( xPendedTicks == ( TickType_t ) 0 ) { - vApplicationTickHook(); + #if ( portUSING_GRANULAR_LOCKS == 1 ) + { + xApplicationTickRequired = pdTRUE; + } + #else + { + vApplicationTickHook(); + } + #endif } else { @@ -5291,15 +5762,34 @@ BaseType_t xTaskIncrementTick( void ) * scheduler is locked. */ #if ( configUSE_TICK_HOOK == 1 ) { - vApplicationTickHook(); + #if ( portUSING_GRANULAR_LOCKS == 1 ) + { + xApplicationTickRequired = pdTRUE; + } + #else + { + vApplicationTickHook(); + } + #endif } - #endif + #endif /* configUSE_TICK_HOOK */ } #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configUSE_TICK_HOOK == 1 ) ) + { + /* In order not to voilate granular lock critical section hierarchy, calling + * the vApplicationTickHook when leaving the kernel critical section. */ + if( xApplicationTickRequired == pdTRUE ) + { + vApplicationTickHook(); + } + } + #endif + traceRETURN_xTaskIncrementTick( xSwitchRequired ); return xSwitchRequired; @@ -5545,13 +6035,11 @@ BaseType_t xTaskIncrementTick( void ) * SMP port. */ configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 ); - /* vTaskSwitchContext() must not be called with a task that has - * preemption disabled. */ - #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) - configASSERT( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ); - #endif - - if( uxSchedulerSuspended != ( UBaseType_t ) 0U ) + if( ( uxSchedulerSuspended != ( UBaseType_t ) 0U ) + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + || ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable != 0U ) + #endif + ) { /* The scheduler is currently suspended or the task * has requested to not be preempted - do not allow @@ -5769,137 +6257,152 @@ void vTaskPlaceOnUnorderedEventList( List_t * pxEventList, BaseType_t xTaskRemoveFromEventList( const List_t * const pxEventList ) { - TCB_t * pxUnblockedTCB; BaseType_t xReturn; - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) - UBaseType_t uxSavedInterruptStatus; - #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ - traceENTER_xTaskRemoveFromEventList( pxEventList ); - #if ( !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) ) - - /* THIS FUNCTION MUST BE CALLED FROM A CRITICAL SECTION. It can also be - * called from a critical section within an ISR. */ - #else /* #if ( ! ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) ) */ + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) /* Lock the kernel data group as we are about to access its members */ - if( portCHECK_IF_IN_ISR() == pdTRUE ) + prvKernelEnterISROnlyCritical(); { - uxSavedInterruptStatus = kernelENTER_CRITICAL_FROM_ISR(); + xReturn = prvTaskRemoveFromEventList( pxEventList ); + } + prvKernelExitISROnlyCritical(); + #else + xReturn = prvTaskRemoveFromEventList( pxEventList ); + #endif + + traceRETURN_xTaskRemoveFromEventList( xReturn ); + return xReturn; +} + +/*-----------------------------------------------------------*/ + +BaseType_t xTaskRemoveFromEventListFromISR( const List_t * const pxEventList ) +{ + BaseType_t xReturn; + + traceENTER_xTaskRemoveFromEventListFromISR( pxEventList ); + + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + { + UBaseType_t uxSavedInterruptStatus; + + /* Lock the kernel data group as we are about to access its members */ + uxSavedInterruptStatus = kernelENTER_CRITICAL_FROM_ISR(); + { + xReturn = prvTaskRemoveFromEventList( pxEventList ); + } + kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); + } + #else /* if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + { + xReturn = prvTaskRemoveFromEventList( pxEventList ); + } + #endif /* if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + + traceRETURN_xTaskRemoveFromEventListFromISR( xReturn ); + return xReturn; +} + +/*-----------------------------------------------------------*/ + +static BaseType_t prvTaskRemoveFromEventList( const List_t * const pxEventList ) +{ + TCB_t * pxUnblockedTCB; + BaseType_t xReturn; + + /* THIS FUNCTION MUST BE CALLED FROM A CRITICAL SECTION. It can also be + * called from a critical section within an ISR. */ + + /* Before proceeding, check if the event list is empty */ + if( listLIST_IS_EMPTY( pxEventList ) == pdFALSE ) + { + /* The event list is sorted in priority order, so the first in the list can + * be removed as it is known to be the highest priority. Remove the TCB from + * the delayed list, and add it to the ready list. + * + * If an event is for a queue that is locked then this function will never + * get called - the lock count on the queue will get modified instead. This + * means exclusive access to the event list is guaranteed here. + * + * This function assumes that a check has already been made to ensure that + * pxEventList is not empty. */ + /* MISRA Ref 11.5.3 [Void pointer assignment] */ + /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-115 */ + /* coverity[misra_c_2012_rule_11_5_violation] */ + pxUnblockedTCB = listGET_OWNER_OF_HEAD_ENTRY( pxEventList ); + configASSERT( pxUnblockedTCB ); + listREMOVE_ITEM( &( pxUnblockedTCB->xEventListItem ) ); + + if( uxSchedulerSuspended == ( UBaseType_t ) 0U ) + { + listREMOVE_ITEM( &( pxUnblockedTCB->xStateListItem ) ); + prvAddTaskToReadyList( pxUnblockedTCB ); + + #if ( configUSE_TICKLESS_IDLE != 0 ) + { + /* If a task is blocked on a kernel object then xNextTaskUnblockTime + * might be set to the blocked task's time out time. If the task is + * unblocked for a reason other than a timeout xNextTaskUnblockTime is + * normally left unchanged, because it is automatically reset to a new + * value when the tick count equals xNextTaskUnblockTime. However if + * tickless idling is used it might be more important to enter sleep mode + * at the earliest possible time - so reset xNextTaskUnblockTime here to + * ensure it is updated at the earliest possible time. */ + prvResetNextTaskUnblockTime(); + } + #endif } else { - uxSavedInterruptStatus = 0; - kernelENTER_CRITICAL(); + /* The delayed and ready lists cannot be accessed, so hold this task + * pending until the scheduler is resumed. */ + listINSERT_END( &( xPendingReadyList ), &( pxUnblockedTCB->xEventListItem ) ); } - /* Before taking the kernel lock, another task/ISR could have already - * emptied the pxEventList. So we insert a check here to see if - * pxEventList is empty before attempting to remove an item from it. */ - if( listLIST_IS_EMPTY( pxEventList ) == pdFALSE ) + #if ( configNUMBER_OF_CORES == 1 ) { - #endif /* #if ( ! ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) ) */ + if( pxUnblockedTCB->uxPriority > pxCurrentTCB->uxPriority ) + { + /* Return true if the task removed from the event list has a higher + * priority than the calling task. This allows the calling task to know if + * it should force a context switch now. */ + xReturn = pdTRUE; - /* The event list is sorted in priority order, so the first in the list can - * be removed as it is known to be the highest priority. Remove the TCB from - * the delayed list, and add it to the ready list. - * - * If an event is for a queue that is locked then this function will never - * get called - the lock count on the queue will get modified instead. This - * means exclusive access to the event list is guaranteed here. - * - * This function assumes that a check has already been made to ensure that - * pxEventList is not empty. */ - /* MISRA Ref 11.5.3 [Void pointer assignment] */ - /* More details at: https://github.com/FreeRTOS/FreeRTOS-Kernel/blob/main/MISRA.md#rule-115 */ - /* coverity[misra_c_2012_rule_11_5_violation] */ - pxUnblockedTCB = listGET_OWNER_OF_HEAD_ENTRY( pxEventList ); - configASSERT( pxUnblockedTCB ); - listREMOVE_ITEM( &( pxUnblockedTCB->xEventListItem ) ); - - if( uxSchedulerSuspended == ( UBaseType_t ) 0U ) - { - listREMOVE_ITEM( &( pxUnblockedTCB->xStateListItem ) ); - prvAddTaskToReadyList( pxUnblockedTCB ); - - #if ( configUSE_TICKLESS_IDLE != 0 ) - { - /* If a task is blocked on a kernel object then xNextTaskUnblockTime - * might be set to the blocked task's time out time. If the task is - * unblocked for a reason other than a timeout xNextTaskUnblockTime is - * normally left unchanged, because it is automatically reset to a new - * value when the tick count equals xNextTaskUnblockTime. However if - * tickless idling is used it might be more important to enter sleep mode - * at the earliest possible time - so reset xNextTaskUnblockTime here to - * ensure it is updated at the earliest possible time. */ - prvResetNextTaskUnblockTime(); + /* Mark that a yield is pending in case the user is not using the + * "xHigherPriorityTaskWoken" parameter to an ISR safe FreeRTOS function. */ + xYieldPendings[ 0 ] = pdTRUE; + } + else + { + xReturn = pdFALSE; + } } - #endif + #else /* #if ( configNUMBER_OF_CORES == 1 ) */ + { + xReturn = pdFALSE; + + #if ( configUSE_PREEMPTION == 1 ) + { + prvYieldForTask( pxUnblockedTCB ); + + if( xYieldPendings[ portGET_CORE_ID() ] != pdFALSE ) + { + xReturn = pdTRUE; + } + } + #endif /* #if ( configUSE_PREEMPTION == 1 ) */ + } + #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ } else { - /* The delayed and ready lists cannot be accessed, so hold this task - * pending until the scheduler is resumed. */ - listINSERT_END( &( xPendingReadyList ), &( pxUnblockedTCB->xEventListItem ) ); - } - - #if ( configNUMBER_OF_CORES == 1 ) - { - if( pxUnblockedTCB->uxPriority > pxCurrentTCB->uxPriority ) - { - /* Return true if the task removed from the event list has a higher - * priority than the calling task. This allows the calling task to know if - * it should force a context switch now. */ - xReturn = pdTRUE; - - /* Mark that a yield is pending in case the user is not using the - * "xHigherPriorityTaskWoken" parameter to an ISR safe FreeRTOS function. */ - xYieldPendings[ 0 ] = pdTRUE; - } - else - { - xReturn = pdFALSE; - } - } - #else /* #if ( configNUMBER_OF_CORES == 1 ) */ - { + /* The pxEventList was emptied before we entered the critical + * section, Nothing to do except return pdFALSE. */ xReturn = pdFALSE; - - #if ( configUSE_PREEMPTION == 1 ) - { - prvYieldForTask( pxUnblockedTCB ); - - if( xYieldPendings[ portGET_CORE_ID() ] != pdFALSE ) - { - xReturn = pdTRUE; - } - } - #endif /* #if ( configUSE_PREEMPTION == 1 ) */ } - #endif /* #if ( configNUMBER_OF_CORES == 1 ) */ - #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) -} -else -{ - /* The pxEventList was emptied before we entered the critical - * section, Nothing to do except return pdFALSE. */ - xReturn = pdFALSE; -} - -/* We are done accessing the kernel data group. Unlock it. */ -if( portCHECK_IF_IN_ISR() == pdTRUE ) -{ - kernelEXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus ); -} -else -{ - kernelEXIT_CRITICAL(); -} - #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ - - traceRETURN_xTaskRemoveFromEventList( xReturn ); return xReturn; } /*-----------------------------------------------------------*/ @@ -5911,9 +6414,12 @@ void vTaskRemoveFromUnorderedEventList( ListItem_t * pxEventListItem, traceENTER_vTaskRemoveFromUnorderedEventList( pxEventListItem, xItemValue ); - /* THIS FUNCTION MUST BE CALLED WITH THE SCHEDULER SUSPENDED. It is used by - * the event flags implementation. */ - configASSERT( uxSchedulerSuspended != ( UBaseType_t ) 0U ); + #if ( !( portUSING_GRANULAR_LOCKS == 1 ) ) + + /* THIS FUNCTION MUST BE CALLED WITH THE SCHEDULER SUSPENDED. It is used by + * the event flags implementation. */ + configASSERT( uxSchedulerSuspended != ( UBaseType_t ) 0U ); + #endif /* #if ( ! ( portUSING_GRANULAR_LOCKS == 1 ) ) */ /* Store the new item value in the event list. */ listSET_LIST_ITEM_VALUE( pxEventListItem, xItemValue | taskEVENT_LIST_ITEM_VALUE_IN_USE ); @@ -5998,7 +6504,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) /* Lock the kernel data group as we are about to access its members */ - kernelENTER_CRITICAL(); + prvKernelEnterISROnlyCritical(); #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ /* For internal use only as it does not use a critical section. */ @@ -6007,7 +6513,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) /* We are done accessing the kernel data group. Unlock it. */ - kernelEXIT_CRITICAL(); + prvKernelExitISROnlyCritical(); #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ traceRETURN_vTaskInternalSetTimeOutState(); @@ -7431,7 +7937,16 @@ STATIC void prvResetNextTaskUnblockTime( void ) { const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); - if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + if( ( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0U ) + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + + /* Task yield can be called in a data group critilcal section. + * Adding a preemption disable check to prevent invalid context + * switch. */ + && ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) && + ( pxCurrentTCBs[ xCoreID ]->uxDeferredStateChange == 0U ) + #endif /* #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + ) { portYIELD(); } @@ -7521,7 +8036,8 @@ STATIC void prvResetNextTaskUnblockTime( void ) * interrupt. Only assert if the critical nesting count is 1 to * protect against recursive calls if the assert function also uses a * critical section. */ - if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 1U ) + if( ( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 1U ) && + ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) ) { portASSERT_IF_IN_ISR(); @@ -7658,7 +8174,7 @@ STATIC void prvResetNextTaskUnblockTime( void ) BaseType_t xYieldCurrentTask; /* Get the xYieldPending status inside the critical section. */ - if( ( xYieldPendings[ xCoreID ] == pdTRUE ) && ( uxSchedulerSuspended == pdFALSE ) + if( ( xYieldPendings[ xCoreID ] == pdTRUE ) #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) && ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) && ( pxCurrentTCBs[ xCoreID ]->uxDeferredStateChange == 0U ) @@ -7800,6 +8316,41 @@ STATIC void prvResetNextTaskUnblockTime( void ) #endif /* #if ( configNUMBER_OF_CORES > 1 ) */ /*-----------------------------------------------------------*/ +/* ISR only critical can only be used when multi-critical section is used. + * Therefore, run state change is not valid due to task can't be requested to yield. */ +static void prvKernelEnterISROnlyCritical( void ) +{ + if( xSchedulerRunning != pdFALSE ) + { + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + + configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ); + + /* Take only the ISR lock, not the task lock. */ + kernelGET_ISR_LOCK( xCoreID ); + + portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + } +} +/*-----------------------------------------------------------*/ + +static void prvKernelExitISROnlyCritical( void ) +{ + if( xSchedulerRunning != pdFALSE ) + { + const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID(); + + configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U ); + + /* Release only the ISR lock. */ + kernelRELEASE_ISR_LOCK( xCoreID ); + + portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID ); + } +} + +/*-----------------------------------------------------------*/ + #if ( configUSE_STATS_FORMATTING_FUNCTIONS > 0 ) STATIC char * prvWriteNameToBuffer( char * pcBuffer, @@ -8730,6 +9281,9 @@ TickType_t uxTaskResetEventItemValue( void ) #else /* #if ( configNUMBER_OF_CORES == 1 ) */ { #if ( configUSE_PREEMPTION == 1 ) + + /* Yield for the unblocked task is required even when scheduler + * is suspended. */ { prvYieldForTask( pxTCB ); @@ -8864,6 +9418,9 @@ TickType_t uxTaskResetEventItemValue( void ) #else /* #if ( configNUMBER_OF_CORES == 1 ) */ { #if ( configUSE_PREEMPTION == 1 ) + + /* Yield for the unblocked task is required even when scheduler + * is suspended. */ { prvYieldForTask( pxTCB ); @@ -9411,3 +9968,32 @@ void vTaskResetState( void ) #endif /* #if ( configGENERATE_RUN_TIME_STATS == 1 ) */ } /*-----------------------------------------------------------*/ + +#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + + BaseType_t xTaskUnlockCanYield( void ) + { + BaseType_t xReturn; + BaseType_t xCoreID = portGET_CORE_ID(); + + /* Current core to call this function should handle the yield request when + * the scheduler is suspended. In vTaskSwitchContext, the core will be blocking + * on TASK spinlocks until scheduler is resumed. */ + if( ( xYieldPendings[ xCoreID ] == pdTRUE ) + #if ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) + && ( pxCurrentTCBs[ xCoreID ]->uxPreemptionDisable == 0U ) + #endif /* ( configUSE_TASK_PREEMPTION_DISABLE == 1 ) */ + ) + { + xReturn = pdTRUE; + } + else + { + xReturn = pdFALSE; + } + + return xReturn; + } + +#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ +/*-----------------------------------------------------------*/ diff --git a/timers.c b/timers.c index 1bc40bc46..b283522cb 100644 --- a/timers.c +++ b/timers.c @@ -149,6 +149,16 @@ PRIVILEGED_DATA static QueueHandle_t xTimerQueue = NULL; PRIVILEGED_DATA static TaskHandle_t xTimerTaskHandle = NULL; + #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) + #ifdef portREMOVE_STATIC_QUALIFIER + PRIVILEGED_DATA portSPINLOCK_TYPE xTimerTaskSpinlock = portINIT_SPINLOCK_STATIC; + PRIVILEGED_DATA portSPINLOCK_TYPE xTimerISRSpinlock = portINIT_SPINLOCK_STATIC; + #else + PRIVILEGED_DATA static portSPINLOCK_TYPE xTimerTaskSpinlock = portINIT_SPINLOCK_STATIC; + PRIVILEGED_DATA static portSPINLOCK_TYPE xTimerISRSpinlock = portINIT_SPINLOCK_STATIC; + #endif + #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */ + /*-----------------------------------------------------------*/ /*