feat(granular_locks): Add granular locking functions

- Updated prvCheckForRunStateChange() for granular locks
- Updated vTaskSuspendAll() and xTaskResumeAll()
    - Now holds the xTaskSpinlock during kernel suspension
    - Increments/decrements xPreemptionDisable. Only yields when 0, thus allowing
    for nested suspensions across different data groups

Co-authored-by: Sudeep Mohanty <sudeep.mohanty@espressif.com>

include/FreeRTOS.h

@@ -465,7 +465,7 @@
     #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) )
         #define portRELEASE_TASK_LOCK( xCoreID )
     #else
-        #error portRELEASE_TASK_LOCK is required in SMP
+        #error portRELEASE_TASK_LOCK is required in SMP without granular locking feature enabled
     #endif
 
 #endif /* portRELEASE_TASK_LOCK */
@@ -475,7 +475,7 @@
     #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) )
         #define portGET_TASK_LOCK( xCoreID )
     #else
-        #error portGET_TASK_LOCK is required in SMP
+        #error portGET_TASK_LOCK is required in SMP without granular locking feature enabled
     #endif
 
 #endif /* portGET_TASK_LOCK */
@@ -485,7 +485,7 @@
     #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) )
         #define portRELEASE_ISR_LOCK( xCoreID )
     #else
-        #error portRELEASE_ISR_LOCK is required in SMP
+        #error portRELEASE_ISR_LOCK is required in SMP without granular locking feature enabled
     #endif
 
 #endif /* portRELEASE_ISR_LOCK */
@@ -495,7 +495,7 @@
     #if ( ( portUSING_GRANULAR_LOCKS == 1 ) || ( configNUMBER_OF_CORES == 1 ) )
         #define portGET_ISR_LOCK( xCoreID )
     #else
-        #error portGET_ISR_LOCK is required in SMP
+        #error portGET_ISR_LOCK is required in SMP without granular locking feature enabled
     #endif
 
 #endif /* portGET_ISR_LOCK */
@@ -503,7 +503,7 @@
 #ifndef portRELEASE_SPINLOCK
 
     #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portRELEASE_SPINLOCK is required for granular locking
+        #error portRELEASE_SPINLOCK is required for SMP with granular locking feature enabled
     #endif
 
 #endif
@@ -511,7 +511,7 @@
 #ifndef portGET_SPINLOCK
 
     #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portGET_SPINLOCK is required for granular locking
+        #error portGET_SPINLOCK is required for SMP with granular locking feature enabled
     #endif
 
 #endif
@@ -540,38 +540,6 @@
 
 #endif
 
-#ifndef portENTER_CRITICAL_DATA_GROUP
-
-    #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portENTER_CRITICAL_DATA_GROUP is required for SMP with granular locking feature enabled
-    #endif
-
-#endif
-
-#ifndef portEXIT_CRITICAL_DATA_GROUP
-
-    #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portEXIT_CRITICAL_DATA_GROUP is required for SMP with granular locking feature enabled
-    #endif
-
-#endif
-
-#ifndef portENTER_CRITICAL_DATA_GROUP_FROM_ISR
-
-    #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portENTER_CRITICAL_DATA_GROUP_FROM_ISR is required for SMP with granular locking feature enabled
-    #endif
-
-#endif
-
-#ifndef portEXIT_CRITICAL_DATA_GROUP_FROM_ISR
-
-    #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portEXIT_CRITICAL_DATA_GROUP_FROM_ISR is required for SMP with granular locking feature enabled
-    #endif
-
-#endif
-
 #ifndef portSPINLOCK_TYPE
 
     #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
@@ -580,82 +548,18 @@
 
 #endif
 
-#ifndef portINIT_EVENT_GROUP_TASK_SPINLOCK
+#ifndef portINIT_SPINLOCK
 
     #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portINIT_EVENT_GROUP_TASK_SPINLOCK is required for granular locking
+        #error portINIT_SPINLOCK is required for SMP with granular locking feature enabled
     #endif
 
 #endif
 
-#ifndef portINIT_EVENT_GROUP_ISR_SPINLOCK
+#ifndef portINIT_SPINLOCK_STATIC
 
     #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portINIT_EVENT_GROUP_ISR_SPINLOCK is required for granular locking
+        #error portINIT_SPINLOCK_STATIC is required for SMP with granular locking feature enabled
-    #endif
-
-#endif
-
-#ifndef portINIT_QUEUE_TASK_SPINLOCK
-
-    #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portINIT_QUEUE_TASK_SPINLOCK is required for granular locking
-    #endif
-
-#endif
-
-#ifndef portINIT_QUEUE_ISR_SPINLOCK
-
-    #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portINIT_QUEUE_ISR_SPINLOCK is required for granular locking
-    #endif
-
-#endif
-
-#ifndef portINIT_STREAM_BUFFER_TASK_SPINLOCK
-
-    #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portINIT_STREAM_BUFFER_TASK_SPINLOCK is required for granular locking
-    #endif
-
-#endif
-
-#ifndef portINIT_STREAM_BUFFER_ISR_SPINLOCK
-
-    #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portINIT_STREAM_BUFFER_ISR_SPINLOCK is required for granular locking
-    #endif
-
-#endif
-
-#ifndef portINIT_KERNEL_TASK_SPINLOCK_STATIC
-
-    #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portINIT_KERNEL_TASK_SPINLOCK_STATIC is required for granular locking
-    #endif
-
-#endif
-
-#ifndef portINIT_KERNEL_ISR_SPINLOCK_STATIC
-
-    #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portINIT_KERNEL_ISR_SPINLOCK_STATIC is required for granular locking
-    #endif
-
-#endif
-
-#ifndef portINIT_TIMERS_TASK_SPINLOCK_STATIC
-
-    #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portINIT_TIMERS_TASK_SPINLOCK_STATIC is required for granular locking
-    #endif
-
-#endif
-
-#ifndef portINIT_TIMERS_ISR_SPINLOCK_STATIC
-
-    #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
-        #error portINIT_TIMERS_ISR_SPINLOCK_STATIC is required for granular locking
     #endif
 
 #endif
@@ -3090,7 +2994,7 @@
     #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */
     #define portTICK_TYPE_SET_INTERRUPT_MASK_FROM_ISR()         portSET_INTERRUPT_MASK_FROM_ISR()
     #define portTICK_TYPE_CLEAR_INTERRUPT_MASK_FROM_ISR( x )    portCLEAR_INTERRUPT_MASK_FROM_ISR( ( x ) )
-#else
+#else /* if ( portTICK_TYPE_IS_ATOMIC == 0 ) */
 
 /* The tick type can be read atomically, so critical sections used when the
  * tick count is returned can be defined away. */

include/task.h

@@ -224,7 +224,7 @@ typedef enum
  * \defgroup taskYIELD taskYIELD
  * \ingroup SchedulerControl
  */
-#define taskYIELD()                          portYIELD()
+#define taskYIELD()                                                 portYIELD()
 
 /**
  * task. h
@@ -238,12 +238,19 @@ typedef enum
  * \defgroup taskENTER_CRITICAL taskENTER_CRITICAL
  * \ingroup SchedulerControl
  */
-#define taskENTER_CRITICAL()                 portENTER_CRITICAL()
+#define taskENTER_CRITICAL()                                        portENTER_CRITICAL()
 #if ( configNUMBER_OF_CORES == 1 )
-    #define taskENTER_CRITICAL_FROM_ISR()    portSET_INTERRUPT_MASK_FROM_ISR()
+    #define taskENTER_CRITICAL_FROM_ISR()                           portSET_INTERRUPT_MASK_FROM_ISR()
 #else
-    #define taskENTER_CRITICAL_FROM_ISR()    portENTER_CRITICAL_FROM_ISR()
+    #define taskENTER_CRITICAL_FROM_ISR()                           portENTER_CRITICAL_FROM_ISR()
 #endif
+#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
+    #define taskLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock )    portLOCK_DATA_GROUP( ( portSPINLOCK_TYPE * ) pxTaskSpinlock, ( portSPINLOCK_TYPE * ) pxISRSpinlock )
+    #define taskLOCK_DATA_GROUP_FROM_ISR( pxISRSpinlock )           portLOCK_DATA_GROUP_FROM_ISR( pxISRSpinlock )
+#else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */
+    #define taskLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock )    taskENTER_CRITICAL()
+    #define taskLOCK_DATA_GROUP_FROM_ISR( pxISRSpinlock )           taskENTER_CRITICAL_FROM_ISR()
+#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */
 
 /**
  * task. h
@@ -257,12 +264,19 @@ typedef enum
  * \defgroup taskEXIT_CRITICAL taskEXIT_CRITICAL
  * \ingroup SchedulerControl
  */
-#define taskEXIT_CRITICAL()                    portEXIT_CRITICAL()
+#define taskEXIT_CRITICAL()                                           portEXIT_CRITICAL()
 #if ( configNUMBER_OF_CORES == 1 )
-    #define taskEXIT_CRITICAL_FROM_ISR( x )    portCLEAR_INTERRUPT_MASK_FROM_ISR( x )
+    #define taskEXIT_CRITICAL_FROM_ISR( x )                           portCLEAR_INTERRUPT_MASK_FROM_ISR( x )
 #else
-    #define taskEXIT_CRITICAL_FROM_ISR( x )    portEXIT_CRITICAL_FROM_ISR( x )
+    #define taskEXIT_CRITICAL_FROM_ISR( x )                           portEXIT_CRITICAL_FROM_ISR( x )
 #endif
+#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
+    #define taskUNLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock )    portUNLOCK_DATA_GROUP( ( portSPINLOCK_TYPE * ) pxTaskSpinlock, ( portSPINLOCK_TYPE * ) pxISRSpinlock )
+    #define taskUNLOCK_DATA_GROUP_FROM_ISR( x, pxISRSpinlock )        portUNLOCK_DATA_GROUP_FROM_ISR( x, pxISRSpinlock )
+#else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */
+    #define taskUNLOCK_DATA_GROUP( pxTaskSpinlock, pxISRSpinlock )    taskEXIT_CRITICAL()
+    #define taskUNLOCK_DATA_GROUP_FROM_ISR( x, pxISRSpinlock )        taskEXIT_CRITICAL_FROM_ISR( x )
+#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */
 
 /**
  * task. h
@@ -294,6 +308,146 @@ typedef enum
 /* Checks if core ID is valid. */
 #define taskVALID_CORE_ID( xCoreID )    ( ( ( ( ( BaseType_t ) 0 <= ( xCoreID ) ) && ( ( xCoreID ) < ( BaseType_t ) configNUMBER_OF_CORES ) ) ) ? ( pdTRUE ) : ( pdFALSE ) )
 
+/**
+ * task. h
+ *
+ * Macro to enter a data group critical section.
+ *
+ * \defgroup taskDATA_GROUP_ENTER_CRITICAL taskDATA_GROUP_ENTER_CRITICAL
+ * \ingroup GranularLocks
+ */
+#if ( portUSING_GRANULAR_LOCKS == 1 )
+    #define taskDATA_GROUP_ENTER_CRITICAL( pxTaskSpinlock, pxISRSpinlock )                \
+    do {                                                                                  \
+        /* Disable preemption to avoid task state changes during the critical section. */ \
+        vTaskPreemptionDisable( NULL );                                                   \
+        {                                                                                 \
+            const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID();                  \
+            /* Task spinlock is always taken first */                                     \
+            portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxTaskSpinlock );          \
+            /* Disable interrupts */                                                      \
+            portDISABLE_INTERRUPTS();                                                     \
+            /* Take the ISR spinlock next */                                              \
+            portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock );           \
+            /* Increment the critical nesting count */                                    \
+            portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID );                              \
+        }                                                                                 \
+    } while( 0 )
+#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */
+
+/**
+ * task. h
+ *
+ * Macro to enter a data group critical section from an interrupt.
+ *
+ * \defgroup taskDATA_GROUP_ENTER_CRITICAL_FROM_ISR taskDATA_GROUP_ENTER_CRITICAL_FROM_ISR
+ * \ingroup GranularLocks
+ */
+#if ( portUSING_GRANULAR_LOCKS == 1 )
+    #define taskDATA_GROUP_ENTER_CRITICAL_FROM_ISR( pxISRSpinlock, puxSavedInterruptStatus ) \
+    do {                                                                                     \
+        *( puxSavedInterruptStatus ) = portSET_INTERRUPT_MASK_FROM_ISR();                    \
+        const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID();                         \
+        /* Take the ISR spinlock */                                                          \
+        portGET_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock );                  \
+        /* Increment the critical nesting count */                                           \
+        portINCREMENT_CRITICAL_NESTING_COUNT( xCoreID );                                     \
+    } while( 0 )
+#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */
+
+/**
+ * task. h
+ *
+ * Macro to exit a data group critical section.
+ *
+ * \defgroup taskDATA_GROUP_EXIT_CRITICAL taskDATA_GROUP_EXIT_CRITICAL
+ * \ingroup GranularLocks
+ */
+#if ( portUSING_GRANULAR_LOCKS == 1 )
+    #define taskDATA_GROUP_EXIT_CRITICAL( pxTaskSpinlock, pxISRSpinlock )        \
+    do {                                                                         \
+        const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID();             \
+        configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U );          \
+        /* Release the ISR spinlock */                                           \
+        portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock );  \
+        /* Release the task spinlock */                                          \
+        portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxTaskSpinlock ); \
+        /* Decrement the critical nesting count */                               \
+        portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID );                         \
+        /* Enable interrupts only if the critical nesting count is 0 */          \
+        if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 )                     \
+        {                                                                        \
+            portENABLE_INTERRUPTS();                                             \
+        }                                                                        \
+        else                                                                     \
+        {                                                                        \
+            mtCOVERAGE_TEST_MARKER();                                            \
+        }                                                                        \
+        /* Re-enable preemption */                                               \
+        vTaskPreemptionEnable( NULL );                                           \
+    } while( 0 )
+#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */
+
+/**
+ * task. h
+ *
+ * Macro to exit a data group critical section from an interrupt.
+ *
+ * \defgroup taskDATA_GROUP_EXIT_CRITICAL_FROM_ISR taskDATA_GROUP_EXIT_CRITICAL_FROM_ISR
+ * \ingroup GranularLocks
+ */
+#if ( portUSING_GRANULAR_LOCKS == 1 )
+    #define taskDATA_GROUP_EXIT_CRITICAL_FROM_ISR( uxSavedInterruptStatus, pxISRSpinlock ) \
+    do {                                                                                   \
+        const BaseType_t xCoreID = ( BaseType_t ) portGET_CORE_ID();                       \
+        configASSERT( portGET_CRITICAL_NESTING_COUNT( xCoreID ) > 0U );                    \
+        /* Decrement the critical nesting count */                                         \
+        portDECREMENT_CRITICAL_NESTING_COUNT( xCoreID );                                   \
+        /* Release the ISR spinlock */                                                     \
+        portRELEASE_SPINLOCK( xCoreID, ( portSPINLOCK_TYPE * ) pxISRSpinlock );            \
+        if( portGET_CRITICAL_NESTING_COUNT( xCoreID ) == 0 )                               \
+        {                                                                                  \
+            portCLEAR_INTERRUPT_MASK_FROM_ISR( uxSavedInterruptStatus );                   \
+        }                                                                                  \
+    } while( 0 )
+#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */
+
+/**
+ * task. h
+ *
+ * Macros to lock a data group (task-level lock only).
+ *
+ * \defgroup taskDATA_GROUP_LOCK taskDATA_GROUP_LOCK
+ * \ingroup GranularLocks
+ */
+#if ( portUSING_GRANULAR_LOCKS == 1 )
+    #define taskDATA_GROUP_LOCK( pxTaskSpinlock )                                              \
+    do {                                                                                       \
+        /* Disable preemption while holding the task spinlock. */                              \
+        vTaskPreemptionDisable( NULL );                                                        \
+        {                                                                                      \
+            portGET_SPINLOCK( portGET_CORE_ID(), ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ); \
+        }                                                                                      \
+    } while( 0 )
+#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */
+
+/**
+ * task. h
+ *
+ * Macros to unlock a data group (task-level lock only).
+ *
+ * \defgroup taskDATA_GROUP_UNLOCK taskDATA_GROUP_UNLOCK
+ * \ingroup GranularLocks
+ */
+#if ( portUSING_GRANULAR_LOCKS == 1 )
+    #define taskDATA_GROUP_UNLOCK( pxTaskSpinlock )                                            \
+    ( {                                                                                        \
+        portRELEASE_SPINLOCK( portGET_CORE_ID(), ( portSPINLOCK_TYPE * ) ( pxTaskSpinlock ) ); \
+        /* Re-enable preemption after releasing the task spinlock. */                          \
+        prvTaskPreemptionEnable( NULL );                                                       \
+    } )
+#endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */
+
 /*-----------------------------------------------------------
 * TASK CREATION API
 *----------------------------------------------------------*/
@@ -1556,6 +1710,23 @@ BaseType_t xTaskResumeFromISR( TaskHandle_t xTaskToResume ) PRIVILEGED_FUNCTION;
     void vTaskPreemptionEnable( const TaskHandle_t xTask );
 #endif
 
+#if ( configUSE_TASK_PREEMPTION_DISABLE == 1 )
+
+/*
+ * THIS FUNCTION MUST NOT BE USED FROM APPLICATION CODE.  IT IS ONLY
+ * INTENDED FOR USE WHEN IMPLEMENTING A PORT OF THE SCHEDULER AND IS
+ * AN INTERFACE WHICH IS FOR THE EXCLUSIVE USE OF THE SCHEDULER.
+ *
+ * @param xTask The handle of the task to enable preemption. Passing NULL
+ * enables preemption for the calling task.
+ *
+ * @return pdTRUE if enabling preemption for the task resulted in a context
+ * switch, otherwise pdFALSE. This is used by the scheduler to determine if a
+ * context switch may be required following the enable.
+ */
+    BaseType_t prvTaskPreemptionEnable( const TaskHandle_t xTask );
+#endif
+
 /*-----------------------------------------------------------
 * SCHEDULER CONTROL
 *----------------------------------------------------------*/
@@ -3825,7 +3996,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC
  * It should be used in the implementation of portENTER_CRITICAL if port is running a
  * multiple core FreeRTOS.
  */
-#if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) || ( configNUMBER_OF_CORES > 1 ) )
+#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
     void vTaskEnterCritical( void );
 #endif
 
@@ -3837,7 +4008,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC
  * It should be used in the implementation of portEXIT_CRITICAL if port is running a
  * multiple core FreeRTOS.
  */
-#if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) || ( configNUMBER_OF_CORES > 1 ) )
+#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
     void vTaskExitCritical( void );
 #endif
 
@@ -3847,7 +4018,7 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC
  * should be used in the implementation of portENTER_CRITICAL_FROM_ISR if port is
  * running a multiple core FreeRTOS.
  */
-#if ( configNUMBER_OF_CORES > 1 )
+#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
     UBaseType_t vTaskEnterCriticalFromISR( void );
 #endif
 
@@ -3857,10 +4028,18 @@ void vTaskInternalSetTimeOutState( TimeOut_t * const pxTimeOut ) PRIVILEGED_FUNC
  * should be used in the implementation of portEXIT_CRITICAL_FROM_ISR if port is
  * running a multiple core FreeRTOS.
  */
-#if ( configNUMBER_OF_CORES > 1 )
+#if !( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
     void vTaskExitCriticalFromISR( UBaseType_t uxSavedInterruptStatus );
 #endif
 
+/*
+ * Checks whether a yield is required after taskUNLOCK_DATA_GROUP() returns.
+ * To be called while data group is locked.
+ */
+#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
+    BaseType_t xTaskUnlockCanYield( void );
+#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */
+
 #if ( portUSING_MPU_WRAPPERS == 1 )
 
 /*

tasks.c

@@ -541,6 +541,11 @@ PRIVILEGED_DATA STATIC volatile configRUN_TIME_COUNTER_TYPE ulTotalRunTime[ conf
 
 #endif
 
+#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
+    PRIVILEGED_DATA static portSPINLOCK_TYPE xTaskSpinlock = portINIT_KERNEL_TASK_SPINLOCK_STATIC;
+    PRIVILEGED_DATA static portSPINLOCK_TYPE xISRSpinlock = portINIT_KERNEL_ISR_SPINLOCK_STATIC;
+#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */
+
 /*-----------------------------------------------------------*/
 
 /* File private functions. --------------------------------*/
@@ -550,14 +555,14 @@ PRIVILEGED_DATA STATIC volatile configRUN_TIME_COUNTER_TYPE ulTotalRunTime[ conf
  */
 STATIC BaseType_t prvCreateIdleTasks( void );
 
-#if ( configNUMBER_OF_CORES > 1 )
+#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) )
 
 /*
  * Checks to see if another task moved the current task out of the ready
  * list while it was waiting to enter a critical section and yields, if so.
  */
     STATIC void prvCheckForRunStateChange( void );
-#endif /* #if ( configNUMBER_OF_CORES > 1 ) */
+#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */
 
 #if ( configNUMBER_OF_CORES > 1 )
 
@@ -807,7 +812,7 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION;
 #endif /* #if ( ( configUSE_TRACE_FACILITY == 1 ) && ( configUSE_STATS_FORMATTING_FUNCTIONS > 0 ) ) */
 /*-----------------------------------------------------------*/
 
-#if ( configNUMBER_OF_CORES > 1 )
+#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) )
     STATIC void prvCheckForRunStateChange( void )
     {
         UBaseType_t uxPrevCriticalNesting;
@@ -871,7 +876,7 @@ STATIC void prvAddNewTaskToReadyList( TCB_t * pxNewTCB ) PRIVILEGED_FUNCTION;
             }
         }
     }
-#endif /* #if ( configNUMBER_OF_CORES > 1 ) */
+#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */
 
 /*-----------------------------------------------------------*/
 
@@ -3958,31 +3963,45 @@ void vTaskSuspendAll( void )
              * do not otherwise exhibit real time behaviour. */
             portSOFTWARE_BARRIER();
 
-            portGET_TASK_LOCK( xCoreID );
+            #if ( portUSING_GRANULAR_LOCKS == 1 )
-
-            /* uxSchedulerSuspended is increased after prvCheckForRunStateChange. The
-             * purpose is to prevent altering the variable when fromISR APIs are readying
-             * it. */
-            if( uxSchedulerSuspended == 0U )
             {
-                prvCheckForRunStateChange();
+                portGET_SPINLOCK( &xTaskSpinlock );
+                portGET_SPINLOCK( &xISRSpinlock );
+
+                /* Increment xPreemptionDisable to prevent preemption and also
+                 * track whether to yield in xTaskResumeAll(). */
+                pxCurrentTCBs[ portGET_CORE_ID() ]->xPreemptionDisable++;
+
+                /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment
+                 * is used to allow calls to vTaskSuspendAll() to nest. */
+                ++uxSchedulerSuspended;
+
+                portRELEASE_SPINLOCK( &xISRSpinlock );
             }
-            else
+            #else /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */
             {
-                mtCOVERAGE_TEST_MARKER();
+                portGET_TASK_LOCK();
+
+                /* uxSchedulerSuspended is increased after prvCheckForRunStateChange. The
+                 * purpose is to prevent altering the variable when fromISR APIs are readying
+                 * it. */
+                if( uxSchedulerSuspended == 0U )
+                {
+                    prvCheckForRunStateChange();
+                }
+                else
+                {
+                    mtCOVERAGE_TEST_MARKER();
+                }
+
+                portGET_ISR_LOCK();
+
+                /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment
+                 * is used to allow calls to vTaskSuspendAll() to nest. */
+                ++uxSchedulerSuspended;
+                portRELEASE_ISR_LOCK();
             }
-
+            #endif /* #if ( portUSING_GRANULAR_LOCKS == 1 ) */
-            /* Query the coreID again as prvCheckForRunStateChange may have
-             * caused the task to get scheduled on a different core. The correct
-             * task lock for the core is acquired in prvCheckForRunStateChange. */
-            xCoreID = ( BaseType_t ) portGET_CORE_ID();
-
-            portGET_ISR_LOCK( xCoreID );
-
-            /* The scheduler is suspended if uxSchedulerSuspended is non-zero. An increment
-             * is used to allow calls to vTaskSuspendAll() to nest. */
-            ++uxSchedulerSuspended;
-            portRELEASE_ISR_LOCK( xCoreID );
 
             portCLEAR_INTERRUPT_MASK( ulState );
         }
@@ -4087,7 +4106,24 @@ BaseType_t xTaskResumeAll( void )
             configASSERT( uxSchedulerSuspended != 0U );
 
             uxSchedulerSuspended = ( UBaseType_t ) ( uxSchedulerSuspended - 1U );
-            portRELEASE_TASK_LOCK( xCoreID );
+            #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
+            {
+                configASSERT( pxCurrentTCBs[ portGET_CORE_ID() ]->xPreemptionDisable > 0 );
+
+                /* Decrement xPreemptionDisable. If 0, it means this we are not
+                 * in a nested suspension scenario, thus this function and yield
+                 * if necessary. */
+                pxCurrentTCBs[ portGET_CORE_ID() ]->xPreemptionDisable--;
+
+                /* Release the kernel's task spinlock that was held throughout
+                 * the kernel suspension. */
+                portRELEASE_SPINLOCK( &xTaskSpinlock );
+            }
+            #else /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */
+            {
+                portRELEASE_TASK_LOCK();
+            }
+            #endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */
 
             if( uxSchedulerSuspended == ( UBaseType_t ) 0U )
             {
@@ -7129,7 +7165,7 @@ STATIC void prvResetNextTaskUnblockTime( void )
 #endif /* #if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) && ( configNUMBER_OF_CORES == 1 ) ) */
 /*-----------------------------------------------------------*/
 
-#if ( configNUMBER_OF_CORES > 1 )
+#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) )
 
     void vTaskEnterCritical( void )
     {
@@ -7178,11 +7214,10 @@ STATIC void prvResetNextTaskUnblockTime( void )
         traceRETURN_vTaskEnterCritical();
     }
 
-#endif /* #if ( configNUMBER_OF_CORES > 1 ) */
+#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */
-
 /*-----------------------------------------------------------*/
 
-#if ( configNUMBER_OF_CORES > 1 )
+#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) )
 
     UBaseType_t vTaskEnterCriticalFromISR( void )
     {
@@ -7212,7 +7247,7 @@ STATIC void prvResetNextTaskUnblockTime( void )
         return uxSavedInterruptStatus;
     }
 
-#endif /* #if ( configNUMBER_OF_CORES > 1 ) */
+#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */
 /*-----------------------------------------------------------*/
 
 #if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) && ( configNUMBER_OF_CORES == 1 ) )
@@ -7260,7 +7295,7 @@ STATIC void prvResetNextTaskUnblockTime( void )
 #endif /* #if ( ( portCRITICAL_NESTING_IN_TCB == 1 ) && ( configNUMBER_OF_CORES == 1 ) ) */
 /*-----------------------------------------------------------*/
 
-#if ( configNUMBER_OF_CORES > 1 )
+#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) )
 
     void vTaskExitCritical( void )
     {
@@ -7320,10 +7355,10 @@ STATIC void prvResetNextTaskUnblockTime( void )
         traceRETURN_vTaskExitCritical();
     }
 
-#endif /* #if ( configNUMBER_OF_CORES > 1 ) */
+#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */
 /*-----------------------------------------------------------*/
 
-#if ( configNUMBER_OF_CORES > 1 )
+#if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) )
 
     void vTaskExitCriticalFromISR( UBaseType_t uxSavedInterruptStatus )
     {
@@ -7366,7 +7401,29 @@ STATIC void prvResetNextTaskUnblockTime( void )
         traceRETURN_vTaskExitCriticalFromISR();
     }
 
-#endif /* #if ( configNUMBER_OF_CORES > 1 ) */
+#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 0 ) && ( configNUMBER_OF_CORES > 1 ) ) */
+/*-----------------------------------------------------------*/
+
+#if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) )
+
+    BaseType_t xTaskUnlockCanYield( void )
+    {
+        BaseType_t xReturn;
+        BaseType_t xCoreID = portGET_CORE_ID();
+
+        if( ( xYieldPendings[ xCoreID ] == pdTRUE ) && ( uxSchedulerSuspended == pdFALSE ) && ( pxCurrentTCBs[ xCoreID ]->xPreemptionDisable == pdFALSE ) )
+        {
+            xReturn = pdTRUE;
+        }
+        else
+        {
+            xReturn = pdFALSE;
+        }
+
+        return xReturn;
+    }
+
+#endif /* #if ( ( portUSING_GRANULAR_LOCKS == 1 ) && ( configNUMBER_OF_CORES > 1 ) ) */
 /*-----------------------------------------------------------*/
 
 #if ( configUSE_STATS_FORMATTING_FUNCTIONS > 0 )