feat(auth): send a request again when its token lapsed in the queue

A request can wait to be served for longer than the access token it was
sent with lives, and comes back refused for a token that was valid when
it left. Uploads that take minutes make that wait ordinary.

- tell an expired token apart from any other refusal by the code the
  API returns, and retry only that one, once
- prefer a token another process left behind over minting a second: it
  saves a round trip, and a rotating provider would retire one that is
  still in use. Assign it rather than read it, since the setter is what
  rewrites the session header the retry will carry
- settle the token before the request so the one compared afterwards is
  the one that was actually sent
- report an overloaded server when the second attempt is refused too,
  and let API errors out of the upload path with their own type

tests/test_token_cache.py

@@ -5,11 +5,11 @@ from unittest.mock import patch
 
 import jwt
 
-from mdrsclient.api.utils import token_check
+from mdrsclient.api.utils import retry_on_expired_token, token_check, token_recover
 from mdrsclient.cache import CacheFile
 from mdrsclient.config import normalize_url
 from mdrsclient.connection import MDRSConnection
-from mdrsclient.exceptions import IllegalArgumentException
+from mdrsclient.exceptions import IllegalArgumentException, ServerBusyException, TokenExpiredException
 from mdrsclient.models import Token
 
 REMOTE = "unittest"
@@ -112,5 +112,87 @@ class TestUrlNormalization(unittest.TestCase):
                 normalize_url(url)
 
 
+class TestExpiredTokenRetry(unittest.TestCase):
+    """A request can wait in the server's queue for longer than its access token lives."""
+
+    def setUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        patcher = patch("mdrsclient.cache.CONFIG_DIRNAME", self.tmp.name)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+        self.addCleanup(self.tmp.cleanup)
+        self.connection = MDRSConnection(REMOTE, "http://localhost:8000/api")
+        self.connection.token = make_token(3600, 86400, "sent")
+
+    def test_a_token_someone_else_refreshed_is_reused_rather_than_replaced(self) -> None:
+        used = self.connection.token
+        rotated = make_token(3600, 86400, "rotated")
+        CacheFile(REMOTE).token = rotated  # another process, while the request waited
+
+        with patch("mdrsclient.api.utils.UsersApi") as users_api:
+            recovered = token_recover(self.connection, used)
+
+        self.assertTrue(recovered)
+        users_api.assert_not_called()
+        self.assertEqual(self.connection.token, rotated)
+        # The retry carries whatever the session header says, not what the cache holds,
+        # so reading the new token through is not enough on its own.
+        self.assertEqual(self.connection.session.headers["Authorization"], f"Bearer {rotated.access}")
+
+    def test_the_token_is_refreshed_when_nobody_else_has(self) -> None:
+        used = self.connection.token
+        rotated = make_token(3600, 86400, "rotated")
+
+        with patch("mdrsclient.api.utils.UsersApi") as users_api:
+            users_api.return_value.tokenRefresh.return_value = rotated
+            recovered = token_recover(self.connection, used)
+
+        self.assertTrue(recovered)
+        users_api.return_value.tokenRefresh.assert_called_once()
+        self.assertEqual(self.connection.token, rotated)
+
+    def test_the_request_is_sent_once_more_and_only_once(self) -> None:
+        calls: list[str] = []
+
+        class Api:
+            def __init__(self, connection: MDRSConnection) -> None:
+                self.connection = connection
+
+            @retry_on_expired_token
+            def send(self) -> str:
+                calls.append("sent")
+                if len(calls) == 1:
+                    raise TokenExpiredException("expired on arrival")
+                return "ok"
+
+        with patch("mdrsclient.api.utils.UsersApi") as users_api:
+            users_api.return_value.tokenRefresh.return_value = make_token(3600, 86400, "rotated")
+            self.assertEqual(Api(self.connection).send(), "ok")
+
+        self.assertEqual(len(calls), 2)
+
+    def test_a_second_refusal_is_reported_as_an_overloaded_server(self) -> None:
+        calls: list[str] = []
+
+        class Api:
+            def __init__(self, connection: MDRSConnection) -> None:
+                self.connection = connection
+
+            @retry_on_expired_token
+            def send(self) -> str:
+                calls.append("sent")
+                raise TokenExpiredException("expired on arrival")
+
+        with patch("mdrsclient.api.utils.UsersApi") as users_api:
+            users_api.return_value.tokenRefresh.return_value = make_token(3600, 86400, "rotated")
+            with self.assertRaises(ServerBusyException) as caught:
+                Api(self.connection).send()
+
+        # Sent twice and no more, and the message names the cause the user can act on
+        # rather than an authentication failure that would send them to the login form.
+        self.assertEqual(len(calls), 2)
+        self.assertIn("overloaded", str(caught.exception))
+
+
 if __name__ == "__main__":
     unittest.main()