[5/5] Refactor PI mutexes internal defintions

Message ID 20191030200052.497-5-adhemerval.zanella@linaro.org
State New
Headers show
Series
  • [1/5] Consolidate futex-internal.h
Related show

Commit Message

Adhemerval Zanella Oct. 30, 2019, 8 p.m.
This patch adds the generic futex_lock_pi and futex_unlock_pi to wrap
around the syscall machinery required to issue the syscall calls. It
simplifies a bit the futex code required to implement PI mutexes.

No function changes, checked on x86_64-linux-gnu.
---
 nptl/pthread_mutex_init.c         | 13 ++---
 nptl/pthread_mutex_lock.c         | 24 +++------
 nptl/pthread_mutex_timedlock.c    | 60 +++++++++------------
 nptl/pthread_mutex_trylock.c      |  8 ++-
 nptl/pthread_mutex_unlock.c       |  6 +--
 sysdeps/nptl/futex-internal.h     | 86 +++++++++++++++++++++++++++++++
 sysdeps/nptl/lowlevellock-futex.h |  9 ++++
 7 files changed, 137 insertions(+), 69 deletions(-)

-- 
2.17.1

Comments

Carlos O'Donell Oct. 30, 2019, 9:08 p.m. | #1
On 10/30/19 4:00 PM, Adhemerval Zanella wrote:
> This patch adds the generic futex_lock_pi and futex_unlock_pi to wrap

> around the syscall machinery required to issue the syscall calls. It

> simplifies a bit the futex code required to implement PI mutexes.

> 


OK for master with minor typos fixed.

Reviewed-by: Carlos O'Donell <carlos@redhat.com>


> No function changes, checked on x86_64-linux-gnu.

> ---

>  nptl/pthread_mutex_init.c         | 13 ++---

>  nptl/pthread_mutex_lock.c         | 24 +++------

>  nptl/pthread_mutex_timedlock.c    | 60 +++++++++------------

>  nptl/pthread_mutex_trylock.c      |  8 ++-

>  nptl/pthread_mutex_unlock.c       |  6 +--

>  sysdeps/nptl/futex-internal.h     | 86 +++++++++++++++++++++++++++++++

>  sysdeps/nptl/lowlevellock-futex.h |  9 ++++

>  7 files changed, 137 insertions(+), 69 deletions(-)

> 

> diff --git a/nptl/pthread_mutex_init.c b/nptl/pthread_mutex_init.c

> index fe4eeee37c..20800b80f5 100644

> --- a/nptl/pthread_mutex_init.c

> +++ b/nptl/pthread_mutex_init.c

> @@ -24,6 +24,7 @@

>  #include "pthreadP.h"

>  #include <atomic.h>

>  #include <pthread-offsets.h>

> +#include <futex-internal.h>


OK.

>  

>  #include <stap-probe.h>

>  

> @@ -37,19 +38,13 @@ static const struct pthread_mutexattr default_mutexattr =

>  static bool

>  prio_inherit_missing (void)

>  {

> -#ifdef __NR_futex

>    static int tpi_supported;

> -  if (__glibc_unlikely (tpi_supported == 0))

> +  if (__glibc_unlikely (atomic_load_relaxed (&tpi_supported) == 0))


OK.

>      {

> -      int lock = 0;

> -      INTERNAL_SYSCALL_DECL (err);

> -      int ret = INTERNAL_SYSCALL (futex, err, 4, &lock, FUTEX_UNLOCK_PI, 0, 0);

> -      assert (INTERNAL_SYSCALL_ERROR_P (ret, err));

> -      tpi_supported = INTERNAL_SYSCALL_ERRNO (ret, err) == ENOSYS ? -1 : 1;

> +      int e = futex_unlock_pi (&(unsigned int){0}, 0);

> +      atomic_store_relaxed (&tpi_supported, e == ENOSYS ? -1 : 1);


OK.

>      }

>    return __glibc_unlikely (tpi_supported < 0);

> -#endif

> -  return true;

>  }

>  

>  int

> diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c

> index ace436d5a6..05bba50666 100644

> --- a/nptl/pthread_mutex_lock.c

> +++ b/nptl/pthread_mutex_lock.c

> @@ -24,7 +24,7 @@

>  #include <not-cancel.h>

>  #include "pthreadP.h"

>  #include <atomic.h>

> -#include <lowlevellock.h>

> +#include <futex-internal.h>

>  #include <stap-probe.h>

>  

>  #ifndef lll_lock_elision

> @@ -416,21 +416,16 @@ __pthread_mutex_lock_full (pthread_mutex_t *mutex)

>  	    int private = (robust

>  			   ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)

>  			   : PTHREAD_MUTEX_PSHARED (mutex));

> -	    INTERNAL_SYSCALL_DECL (__err);

> -	    int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,

> -				      __lll_private_flag (FUTEX_LOCK_PI,

> -							  private), 1, 0);

> -

> -	    if (INTERNAL_SYSCALL_ERROR_P (e, __err)

> -		&& (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH

> -		    || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))

> +	    int e = futex_lock_pi ((unsigned int *) &mutex->__data.__lock,

> +				   NULL, private);

> +	    if (e == ESRCH || e == EDEADLK)


OK.

>  	      {

> -		assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK

> +		assert (e != EDEADLK

>  			|| (kind != PTHREAD_MUTEX_ERRORCHECK_NP

>  			    && kind != PTHREAD_MUTEX_RECURSIVE_NP));

>  		/* ESRCH can happen only for non-robust PI mutexes where

>  		   the owner of the lock died.  */

> -		assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);

> +		assert (e != ESRCH || !robust);

>  

>  		/* Delay the thread indefinitely.  */

>  		while (1)

> @@ -479,11 +474,8 @@ __pthread_mutex_lock_full (pthread_mutex_t *mutex)

>  	    /* This mutex is now not recoverable.  */

>  	    mutex->__data.__count = 0;

>  

> -	    INTERNAL_SYSCALL_DECL (__err);

> -	    INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,

> -			      __lll_private_flag (FUTEX_UNLOCK_PI,

> -						  PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),

> -			      0, 0);

> +	    futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,

> +			     PTHREAD_ROBUST_MUTEX_PSHARED (mutex));


OK.

>  

>  	    /* To the kernel, this will be visible after the kernel has

>  	       acquired the mutex in the syscall.  */

> diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c

> index 490064d8cf..112175eb11 100644

> --- a/nptl/pthread_mutex_timedlock.c

> +++ b/nptl/pthread_mutex_timedlock.c

> @@ -25,6 +25,7 @@

>  #include <atomic.h>

>  #include <lowlevellock.h>

>  #include <not-cancel.h>

> +#include <futex-internal.h>

>  

>  #include <stap-probe.h>

>  

> @@ -377,39 +378,29 @@ __pthread_mutex_clocklock_common (pthread_mutex_t *mutex,

>  	    int private = (robust

>  			   ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)

>  			   : PTHREAD_MUTEX_PSHARED (mutex));

> -	    INTERNAL_SYSCALL_DECL (__err);

> -

> -	    int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,

> -				      __lll_private_flag (FUTEX_LOCK_PI,

> -							  private), 1,

> -				      abstime);

> -	    if (INTERNAL_SYSCALL_ERROR_P (e, __err))

> +	    int e = futex_lock_pi ((unsigned int *) &mutex->__data.__lock,

> +				   abstime, private);

> +	    if (e == ETIMEDOUT)

> +	      return ETIMEDOUT;

> +	    else if (e == ESRCH || e == EDEADLK)


OK.

>  	      {

> -		if (INTERNAL_SYSCALL_ERRNO (e, __err) == ETIMEDOUT)

> -		  return ETIMEDOUT;

> -

> -		if (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH

> -		    || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK)

> -		  {

> -		    assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK

> -			    || (kind != PTHREAD_MUTEX_ERRORCHECK_NP

> -				&& kind != PTHREAD_MUTEX_RECURSIVE_NP));

> -		    /* ESRCH can happen only for non-robust PI mutexes where

> -		       the owner of the lock died.  */

> -		    assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH

> -			    || !robust);

> -

> -		    /* Delay the thread until the timeout is reached.

> -		       Then return ETIMEDOUT.  */

> -		    do

> -		      e = lll_timedwait (&(int){0}, 0, clockid, abstime,

> -					 private);

> -		    while (e != ETIMEDOUT);

> -		    return ETIMEDOUT;

> -		  }

> -

> -		return INTERNAL_SYSCALL_ERRNO (e, __err);

> +		assert (e != EDEADLK

> +			|| (kind != PTHREAD_MUTEX_ERRORCHECK_NP

> +			   && kind != PTHREAD_MUTEX_RECURSIVE_NP));

> +		/* ESRCH can happen only for non-robust PI mutexes where

> +		   the owner of the lock died.  */

> +		assert (e != ESRCH || !robust);

> +

> +		/* Delay the thread until the timeout is reached. Then return

> +		   ETIMEDOUT.  */

> +		do

> +		  e = lll_timedwait (&(int){0}, 0, clockid, abstime,

> +				     private);

> +		while (e != ETIMEDOUT);

> +		return ETIMEDOUT;


OK.

>  	      }

> +	    else if (e != 0)

> +	      return e;

>  

>  	    oldval = mutex->__data.__lock;

>  

> @@ -447,11 +438,8 @@ __pthread_mutex_clocklock_common (pthread_mutex_t *mutex,

>  	    /* This mutex is now not recoverable.  */

>  	    mutex->__data.__count = 0;

>  

> -	    INTERNAL_SYSCALL_DECL (__err);

> -	    INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,

> -			      __lll_private_flag (FUTEX_UNLOCK_PI,

> -						  PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),

> -			      0, 0);

> +	    futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,

> +			     PTHREAD_ROBUST_MUTEX_PSHARED (mutex));


OK.

>  

>  	    /* To the kernel, this will be visible after the kernel has

>  	       acquired the mutex in the syscall.  */

> diff --git a/nptl/pthread_mutex_trylock.c b/nptl/pthread_mutex_trylock.c

> index 87e87c013a..d24bb58a8b 100644

> --- a/nptl/pthread_mutex_trylock.c

> +++ b/nptl/pthread_mutex_trylock.c

> @@ -21,6 +21,7 @@

>  #include <stdlib.h>

>  #include "pthreadP.h"

>  #include <lowlevellock.h>

> +#include <futex-internal.h>

>  

>  #ifndef lll_trylock_elision

>  #define lll_trylock_elision(a,t) lll_trylock(a)

> @@ -346,11 +347,8 @@ __pthread_mutex_trylock (pthread_mutex_t *mutex)

>  	    /* This mutex is now not recoverable.  */

>  	    mutex->__data.__count = 0;

>  

> -	    INTERNAL_SYSCALL_DECL (__err);

> -	    INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,

> -			      __lll_private_flag (FUTEX_UNLOCK_PI,

> -						  PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),

> -			      0, 0);

> +	    futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,

> +			     PTHREAD_ROBUST_MUTEX_PSHARED (mutex));

>  


OK.

>  	    /* To the kernel, this will be visible after the kernel has

>  	       acquired the mutex in the syscall.  */

> diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c

> index 71038f92e4..53f8b868e4 100644

> --- a/nptl/pthread_mutex_unlock.c

> +++ b/nptl/pthread_mutex_unlock.c

> @@ -22,6 +22,7 @@

>  #include "pthreadP.h"

>  #include <lowlevellock.h>

>  #include <stap-probe.h>

> +#include <futex-internal.h>

>  

>  #ifndef lll_unlock_elision

>  #define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })

> @@ -277,9 +278,8 @@ __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)

>  	  if (((l & FUTEX_WAITERS) != 0)

>  	      || (l != THREAD_GETMEM (THREAD_SELF, tid)))

>  	    {

> -	      INTERNAL_SYSCALL_DECL (__err);

> -	      INTERNAL_SYSCALL (futex, __err, 2, &mutex->__data.__lock,

> -				__lll_private_flag (FUTEX_UNLOCK_PI, private));

> +	      futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,

> +			       private);


OK.

>  	      break;

>  	    }

>  	}

> diff --git a/sysdeps/nptl/futex-internal.h b/sysdeps/nptl/futex-internal.h

> index 76921466f0..2a8e3fb323 100644

> --- a/sysdeps/nptl/futex-internal.h

> +++ b/sysdeps/nptl/futex-internal.h

> @@ -381,4 +381,90 @@ futex_wake (unsigned int* futex_word, int processes_to_wake, int private)

>      }

>  }

>  

> +/* The operation checks the value of the futer, if the value is 0, then the


s/futer/futex/g

> +   it is atomically set to the caller's thread ID.  If the futex value is

> +   nonzero, it is atomically sets the FUTEX_WAITERS bit, which signals wrt


s/it is/it/g

> +   other futex owner that it cannot unlock the futex in user space


s/space/space by/g

> +   atomically by setting its value to 0.

> +

> +   If more than one wait operations is issued, the enqueueing of the waiters

> +   are done in descending priority order.

> +

> +   The ABSTIME arguments provides an absolute timeout (measured against the

> +   CLOCK_REALTIME clock).  If TIMEOUT is NULL, the operation will block

> +   indefinitely.

> +

> +   Returns:

> +

> +     - 0 if woken by a PI unlock operation or spuriously.

> +     - EAGAIN if the futex owner thread ID is about to exit, but has not yet

> +       handled the state cleanup.

> +     - EDEADLK if the futex is already locked by the caller.

> +     - ESRCH if the thread ID int he futex does not exist.

> +     - EINVAL is the state is corrupted or if there is a waiter on the

> +       futex.

> +     - ETIMEDOUT if the ABSTIME expires.

> +*/

> +static __always_inline int

> +futex_lock_pi (unsigned int *futex_word, const struct timespec *abstime,

> +	       int private)

> +{

> +  int err = lll_futex_timed_lock_pi (futex_word, abstime, private);


OK.

> +  switch (err)

> +    {

> +    case 0:

> +    case -EAGAIN:

> +    case -EINTR:

> +    case -ETIMEDOUT:

> +    case -ESRCH:

> +    case -EDEADLK:

> +    case -EINVAL: /* This indicates either state corruption or that the kernel

> +		     found a waiter on futex address which is waiting via

> +		     FUTEX_WAIT or FUTEX_WAIT_BITSET.  This is reported on

> +		     some futex_lock_pi usage (pthread_mutex_timedlock for

> +		     instance).  */

> +      return -err;

> +

> +    case -EFAULT: /* Must have been caused by a glibc or application bug.  */

> +    case -ENOSYS: /* Must have been caused by a glibc bug.  */

> +    /* No other errors are documented at this time.  */

> +    default:

> +      futex_fatal_error ();


OK.

> +    }

> +}

> +

> +/* Wakes the top priority waiter that called a futex_lock_pi operation on

> +   the futex.

> +

> +   Returns the same values as futex_lock_pi under those same conditions;

> +   additionally, returns EPERM when the caller is not allowed to attach

> +   itself to the futex.  */

> +static __always_inline int

> +futex_unlock_pi (unsigned int *futex_word, int private)

> +{

> +  int err = lll_futex_timed_unlock_pi (futex_word, private);


OK.

> +  switch (err)

> +    {

> +    case 0:

> +    case -EAGAIN:

> +    case -EINTR:

> +    case -ETIMEDOUT:

> +    case -ESRCH:

> +    case -EDEADLK:

> +    case -ENOSYS:

> +    case -EPERM:  /*  The caller is not allowed to attach itself to the futex.

> +		      Used to check if PI futexes are supported by the

> +		      kernel.  */

> +      return -err;

> +

> +    case -EINVAL: /* Either due to wrong alignment or due to the timeout not

> +		     being normalized.  Must have been caused by a glibc or

> +		     application bug.  */

> +    case -EFAULT: /* Must have been caused by a glibc or application bug.  */

> +    /* No other errors are documented at this time.  */

> +    default:

> +      futex_fatal_error ();

> +    }

> +}

> +

>  #endif  /* futex-internal.h */

> diff --git a/sysdeps/nptl/lowlevellock-futex.h b/sysdeps/nptl/lowlevellock-futex.h

> index 392277e8a5..ff0fd4edc4 100644

> --- a/sysdeps/nptl/lowlevellock-futex.h

> +++ b/sysdeps/nptl/lowlevellock-futex.h

> @@ -140,6 +140,15 @@

>  

>  

>  /* Priority Inheritance support.  */

> +#define lll_futex_timed_lock_pi(futexp, abstime, private) 		\

> +  lll_futex_syscall (4, futexp,						\

> +		     __lll_private_flag (FUTEX_LOCK_PI, private),	\

> +		     0, abstime)

> +

> +#define lll_futex_timed_unlock_pi(futexp, private) 			\

> +  lll_futex_syscall (4, futexp,						\

> +		     __lll_private_flag (FUTEX_UNLOCK_PI, private),	\

> +		     0, 0)


OK.

>  

>  /* Like lll_futex_wait (FUTEXP, VAL, PRIVATE) but with the expectation

>     that lll_futex_cmp_requeue_pi (FUTEXP, _, _, MUTEX, _, PRIVATE) will

> 



-- 
Cheers,
Carlos.

Patch

diff --git a/nptl/pthread_mutex_init.c b/nptl/pthread_mutex_init.c
index fe4eeee37c..20800b80f5 100644
--- a/nptl/pthread_mutex_init.c
+++ b/nptl/pthread_mutex_init.c
@@ -24,6 +24,7 @@ 
 #include "pthreadP.h"
 #include <atomic.h>
 #include <pthread-offsets.h>
+#include <futex-internal.h>
 
 #include <stap-probe.h>
 
@@ -37,19 +38,13 @@  static const struct pthread_mutexattr default_mutexattr =
 static bool
 prio_inherit_missing (void)
 {
-#ifdef __NR_futex
   static int tpi_supported;
-  if (__glibc_unlikely (tpi_supported == 0))
+  if (__glibc_unlikely (atomic_load_relaxed (&tpi_supported) == 0))
     {
-      int lock = 0;
-      INTERNAL_SYSCALL_DECL (err);
-      int ret = INTERNAL_SYSCALL (futex, err, 4, &lock, FUTEX_UNLOCK_PI, 0, 0);
-      assert (INTERNAL_SYSCALL_ERROR_P (ret, err));
-      tpi_supported = INTERNAL_SYSCALL_ERRNO (ret, err) == ENOSYS ? -1 : 1;
+      int e = futex_unlock_pi (&(unsigned int){0}, 0);
+      atomic_store_relaxed (&tpi_supported, e == ENOSYS ? -1 : 1);
     }
   return __glibc_unlikely (tpi_supported < 0);
-#endif
-  return true;
 }
 
 int
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index ace436d5a6..05bba50666 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -24,7 +24,7 @@ 
 #include <not-cancel.h>
 #include "pthreadP.h"
 #include <atomic.h>
-#include <lowlevellock.h>
+#include <futex-internal.h>
 #include <stap-probe.h>
 
 #ifndef lll_lock_elision
@@ -416,21 +416,16 @@  __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 	    int private = (robust
 			   ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 			   : PTHREAD_MUTEX_PSHARED (mutex));
-	    INTERNAL_SYSCALL_DECL (__err);
-	    int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
-				      __lll_private_flag (FUTEX_LOCK_PI,
-							  private), 1, 0);
-
-	    if (INTERNAL_SYSCALL_ERROR_P (e, __err)
-		&& (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
-		    || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
+	    int e = futex_lock_pi ((unsigned int *) &mutex->__data.__lock,
+				   NULL, private);
+	    if (e == ESRCH || e == EDEADLK)
 	      {
-		assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
+		assert (e != EDEADLK
 			|| (kind != PTHREAD_MUTEX_ERRORCHECK_NP
 			    && kind != PTHREAD_MUTEX_RECURSIVE_NP));
 		/* ESRCH can happen only for non-robust PI mutexes where
 		   the owner of the lock died.  */
-		assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
+		assert (e != ESRCH || !robust);
 
 		/* Delay the thread indefinitely.  */
 		while (1)
@@ -479,11 +474,8 @@  __pthread_mutex_lock_full (pthread_mutex_t *mutex)
 	    /* This mutex is now not recoverable.  */
 	    mutex->__data.__count = 0;
 
-	    INTERNAL_SYSCALL_DECL (__err);
-	    INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
-			      __lll_private_flag (FUTEX_UNLOCK_PI,
-						  PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
-			      0, 0);
+	    futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
+			     PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 
 	    /* To the kernel, this will be visible after the kernel has
 	       acquired the mutex in the syscall.  */
diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c
index 490064d8cf..112175eb11 100644
--- a/nptl/pthread_mutex_timedlock.c
+++ b/nptl/pthread_mutex_timedlock.c
@@ -25,6 +25,7 @@ 
 #include <atomic.h>
 #include <lowlevellock.h>
 #include <not-cancel.h>
+#include <futex-internal.h>
 
 #include <stap-probe.h>
 
@@ -377,39 +378,29 @@  __pthread_mutex_clocklock_common (pthread_mutex_t *mutex,
 	    int private = (robust
 			   ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
 			   : PTHREAD_MUTEX_PSHARED (mutex));
-	    INTERNAL_SYSCALL_DECL (__err);
-
-	    int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
-				      __lll_private_flag (FUTEX_LOCK_PI,
-							  private), 1,
-				      abstime);
-	    if (INTERNAL_SYSCALL_ERROR_P (e, __err))
+	    int e = futex_lock_pi ((unsigned int *) &mutex->__data.__lock,
+				   abstime, private);
+	    if (e == ETIMEDOUT)
+	      return ETIMEDOUT;
+	    else if (e == ESRCH || e == EDEADLK)
 	      {
-		if (INTERNAL_SYSCALL_ERRNO (e, __err) == ETIMEDOUT)
-		  return ETIMEDOUT;
-
-		if (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
-		    || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK)
-		  {
-		    assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
-			    || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
-				&& kind != PTHREAD_MUTEX_RECURSIVE_NP));
-		    /* ESRCH can happen only for non-robust PI mutexes where
-		       the owner of the lock died.  */
-		    assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH
-			    || !robust);
-
-		    /* Delay the thread until the timeout is reached.
-		       Then return ETIMEDOUT.  */
-		    do
-		      e = lll_timedwait (&(int){0}, 0, clockid, abstime,
-					 private);
-		    while (e != ETIMEDOUT);
-		    return ETIMEDOUT;
-		  }
-
-		return INTERNAL_SYSCALL_ERRNO (e, __err);
+		assert (e != EDEADLK
+			|| (kind != PTHREAD_MUTEX_ERRORCHECK_NP
+			   && kind != PTHREAD_MUTEX_RECURSIVE_NP));
+		/* ESRCH can happen only for non-robust PI mutexes where
+		   the owner of the lock died.  */
+		assert (e != ESRCH || !robust);
+
+		/* Delay the thread until the timeout is reached. Then return
+		   ETIMEDOUT.  */
+		do
+		  e = lll_timedwait (&(int){0}, 0, clockid, abstime,
+				     private);
+		while (e != ETIMEDOUT);
+		return ETIMEDOUT;
 	      }
+	    else if (e != 0)
+	      return e;
 
 	    oldval = mutex->__data.__lock;
 
@@ -447,11 +438,8 @@  __pthread_mutex_clocklock_common (pthread_mutex_t *mutex,
 	    /* This mutex is now not recoverable.  */
 	    mutex->__data.__count = 0;
 
-	    INTERNAL_SYSCALL_DECL (__err);
-	    INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
-			      __lll_private_flag (FUTEX_UNLOCK_PI,
-						  PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
-			      0, 0);
+	    futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
+			     PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 
 	    /* To the kernel, this will be visible after the kernel has
 	       acquired the mutex in the syscall.  */
diff --git a/nptl/pthread_mutex_trylock.c b/nptl/pthread_mutex_trylock.c
index 87e87c013a..d24bb58a8b 100644
--- a/nptl/pthread_mutex_trylock.c
+++ b/nptl/pthread_mutex_trylock.c
@@ -21,6 +21,7 @@ 
 #include <stdlib.h>
 #include "pthreadP.h"
 #include <lowlevellock.h>
+#include <futex-internal.h>
 
 #ifndef lll_trylock_elision
 #define lll_trylock_elision(a,t) lll_trylock(a)
@@ -346,11 +347,8 @@  __pthread_mutex_trylock (pthread_mutex_t *mutex)
 	    /* This mutex is now not recoverable.  */
 	    mutex->__data.__count = 0;
 
-	    INTERNAL_SYSCALL_DECL (__err);
-	    INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
-			      __lll_private_flag (FUTEX_UNLOCK_PI,
-						  PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
-			      0, 0);
+	    futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
+			     PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
 
 	    /* To the kernel, this will be visible after the kernel has
 	       acquired the mutex in the syscall.  */
diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c
index 71038f92e4..53f8b868e4 100644
--- a/nptl/pthread_mutex_unlock.c
+++ b/nptl/pthread_mutex_unlock.c
@@ -22,6 +22,7 @@ 
 #include "pthreadP.h"
 #include <lowlevellock.h>
 #include <stap-probe.h>
+#include <futex-internal.h>
 
 #ifndef lll_unlock_elision
 #define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })
@@ -277,9 +278,8 @@  __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
 	  if (((l & FUTEX_WAITERS) != 0)
 	      || (l != THREAD_GETMEM (THREAD_SELF, tid)))
 	    {
-	      INTERNAL_SYSCALL_DECL (__err);
-	      INTERNAL_SYSCALL (futex, __err, 2, &mutex->__data.__lock,
-				__lll_private_flag (FUTEX_UNLOCK_PI, private));
+	      futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
+			       private);
 	      break;
 	    }
 	}
diff --git a/sysdeps/nptl/futex-internal.h b/sysdeps/nptl/futex-internal.h
index 76921466f0..2a8e3fb323 100644
--- a/sysdeps/nptl/futex-internal.h
+++ b/sysdeps/nptl/futex-internal.h
@@ -381,4 +381,90 @@  futex_wake (unsigned int* futex_word, int processes_to_wake, int private)
     }
 }
 
+/* The operation checks the value of the futer, if the value is 0, then the
+   it is atomically set to the caller's thread ID.  If the futex value is
+   nonzero, it is atomically sets the FUTEX_WAITERS bit, which signals wrt
+   other futex owner that it cannot unlock the futex in user space
+   atomically by setting its value to 0.
+
+   If more than one wait operations is issued, the enqueueing of the waiters
+   are done in descending priority order.
+
+   The ABSTIME arguments provides an absolute timeout (measured against the
+   CLOCK_REALTIME clock).  If TIMEOUT is NULL, the operation will block
+   indefinitely.
+
+   Returns:
+
+     - 0 if woken by a PI unlock operation or spuriously.
+     - EAGAIN if the futex owner thread ID is about to exit, but has not yet
+       handled the state cleanup.
+     - EDEADLK if the futex is already locked by the caller.
+     - ESRCH if the thread ID int he futex does not exist.
+     - EINVAL is the state is corrupted or if there is a waiter on the
+       futex.
+     - ETIMEDOUT if the ABSTIME expires.
+*/
+static __always_inline int
+futex_lock_pi (unsigned int *futex_word, const struct timespec *abstime,
+	       int private)
+{
+  int err = lll_futex_timed_lock_pi (futex_word, abstime, private);
+  switch (err)
+    {
+    case 0:
+    case -EAGAIN:
+    case -EINTR:
+    case -ETIMEDOUT:
+    case -ESRCH:
+    case -EDEADLK:
+    case -EINVAL: /* This indicates either state corruption or that the kernel
+		     found a waiter on futex address which is waiting via
+		     FUTEX_WAIT or FUTEX_WAIT_BITSET.  This is reported on
+		     some futex_lock_pi usage (pthread_mutex_timedlock for
+		     instance).  */
+      return -err;
+
+    case -EFAULT: /* Must have been caused by a glibc or application bug.  */
+    case -ENOSYS: /* Must have been caused by a glibc bug.  */
+    /* No other errors are documented at this time.  */
+    default:
+      futex_fatal_error ();
+    }
+}
+
+/* Wakes the top priority waiter that called a futex_lock_pi operation on
+   the futex.
+
+   Returns the same values as futex_lock_pi under those same conditions;
+   additionally, returns EPERM when the caller is not allowed to attach
+   itself to the futex.  */
+static __always_inline int
+futex_unlock_pi (unsigned int *futex_word, int private)
+{
+  int err = lll_futex_timed_unlock_pi (futex_word, private);
+  switch (err)
+    {
+    case 0:
+    case -EAGAIN:
+    case -EINTR:
+    case -ETIMEDOUT:
+    case -ESRCH:
+    case -EDEADLK:
+    case -ENOSYS:
+    case -EPERM:  /*  The caller is not allowed to attach itself to the futex.
+		      Used to check if PI futexes are supported by the
+		      kernel.  */
+      return -err;
+
+    case -EINVAL: /* Either due to wrong alignment or due to the timeout not
+		     being normalized.  Must have been caused by a glibc or
+		     application bug.  */
+    case -EFAULT: /* Must have been caused by a glibc or application bug.  */
+    /* No other errors are documented at this time.  */
+    default:
+      futex_fatal_error ();
+    }
+}
+
 #endif  /* futex-internal.h */
diff --git a/sysdeps/nptl/lowlevellock-futex.h b/sysdeps/nptl/lowlevellock-futex.h
index 392277e8a5..ff0fd4edc4 100644
--- a/sysdeps/nptl/lowlevellock-futex.h
+++ b/sysdeps/nptl/lowlevellock-futex.h
@@ -140,6 +140,15 @@ 
 
 
 /* Priority Inheritance support.  */
+#define lll_futex_timed_lock_pi(futexp, abstime, private) 		\
+  lll_futex_syscall (4, futexp,						\
+		     __lll_private_flag (FUTEX_LOCK_PI, private),	\
+		     0, abstime)
+
+#define lll_futex_timed_unlock_pi(futexp, private) 			\
+  lll_futex_syscall (4, futexp,						\
+		     __lll_private_flag (FUTEX_UNLOCK_PI, private),	\
+		     0, 0)
 
 /* Like lll_futex_wait (FUTEXP, VAL, PRIVATE) but with the expectation
    that lll_futex_cmp_requeue_pi (FUTEXP, _, _, MUTEX, _, PRIVATE) will