From 4d4d80c1d076dcb4e051c969ae4b66557d3856b8 Mon Sep 17 00:00:00 2001
From: Ben Darnell <ben@bendarnell.com>
Date: Thu, 8 Jun 2023 22:52:19 -0400
Subject: [PATCH] *: Adapt to deprecation of datetime utc methods

Python 3.12 deprecates the utcnow and utcfromtimestamp methods and
discourages the use of naive datetimes to represent UTC. This was
previously the main way that Tornado used datetimes (since it was
the only option available in Python 2 before the introduction
of datetime.timezone.utc in Python 3.2).

- httpclient_test: Test-only change to test that both kinds of datetimes
  are supported in If-Modified-Since (this just calls
  httputil.format_timestamp)
- httputil: No functional changes, but format_timestamp's
  support for both naive and aware datetimes is now tested.
- locale: format_timestamp now supports aware datetimes (in
  addition to the existing support for naive datetimes).
- web: Cookie expirations internally use aware datetimes.
  StaticFileHandler.get_modified_time now supports both and the
  standard implementation returns aware.

It feels fragile that "naive" and "aware" datetimes are not distinct
types but subject to data-dependent behavior. This change uses
"aware" datetimes throughout Tornado, but some operations (comparisons
and subtraction) fail with mixed datetime types and if I missed any
in this change may cause errors if naive datetimes were used (where
previously naive datetimes would have been required). But that's
apparently the API we have to work with.
---
 tornado/httputil.py             |  3 +-
 tornado/locale.py               | 12 +++--
 tornado/test/httpclient_test.py | 12 ++++-
 tornado/test/httputil_test.py   | 26 +++++++++-
 tornado/test/locale_test.py     | 88 ++++++++++++++++++---------------
 tornado/test/web_test.py        | 25 +++++-----
 tornado/web.py                  | 28 +++++++----
 7 files changed, 126 insertions(+), 68 deletions(-)

diff --git a/tornado/httputil.py b/tornado/httputil.py
index 9c341d47cc..b21d8046c4 100644
--- a/tornado/httputil.py
+++ b/tornado/httputil.py
@@ -856,7 +856,8 @@ def format_timestamp(
 
     The argument may be a numeric timestamp as returned by `time.time`,
     a time tuple as returned by `time.gmtime`, or a `datetime.datetime`
-    object.
+    object. Naive `datetime.datetime` objects are assumed to represent
+    UTC; aware objects are converted to UTC before formatting.
 
     >>> format_timestamp(1359312200)
     'Sun, 27 Jan 2013 18:43:20 GMT'
diff --git a/tornado/locale.py b/tornado/locale.py
index 55072af28d..c5526703b1 100644
--- a/tornado/locale.py
+++ b/tornado/locale.py
@@ -333,7 +333,7 @@ def format_date(
         shorter: bool = False,
         full_format: bool = False,
     ) -> str:
-        """Formats the given date (which should be GMT).
+        """Formats the given date.
 
         By default, we return a relative time (e.g., "2 minutes ago"). You
         can return an absolute date string with ``relative=False``.
@@ -343,10 +343,16 @@ def format_date(
 
         This method is primarily intended for dates in the past.
         For dates in the future, we fall back to full format.
+
+        .. versionchanged:: 6.4
+           Aware `datetime.datetime` objects are now supported (naive
+           datetimes are still assumed to be UTC).
         """
         if isinstance(date, (int, float)):
-            date = datetime.datetime.utcfromtimestamp(date)
-        now = datetime.datetime.utcnow()
+            date = datetime.datetime.fromtimestamp(date, datetime.timezone.utc)
+        if date.tzinfo is None:
+            date = date.replace(tzinfo=datetime.timezone.utc)
+        now = datetime.datetime.now(datetime.timezone.utc)
         if date > now:
             if relative and (date - now).seconds < 60:
                 # Due to click skew, things are some things slightly
diff --git a/tornado/test/httpclient_test.py b/tornado/test/httpclient_test.py
index a71ec0afb6..a41040e64a 100644
--- a/tornado/test/httpclient_test.py
+++ b/tornado/test/httpclient_test.py
@@ -28,7 +28,7 @@
 from tornado.log import gen_log, app_log
 from tornado import netutil
 from tornado.testing import AsyncHTTPTestCase, bind_unused_port, gen_test, ExpectLog
-from tornado.test.util import skipOnTravis
+from tornado.test.util import skipOnTravis, ignore_deprecation
 from tornado.web import Application, RequestHandler, url
 from tornado.httputil import format_timestamp, HTTPHeaders
 
@@ -887,7 +887,15 @@ def test_body_setter(self):
         self.assertEqual(request.body, utf8("foo"))
 
     def test_if_modified_since(self):
-        http_date = datetime.datetime.utcnow()
+        http_date = datetime.datetime.now(datetime.timezone.utc)
+        request = HTTPRequest("http://example.com", if_modified_since=http_date)
+        self.assertEqual(
+            request.headers, {"If-Modified-Since": format_timestamp(http_date)}
+        )
+
+    def test_if_modified_since_naive_deprecated(self):
+        with ignore_deprecation():
+            http_date = datetime.datetime.utcnow()
         request = HTTPRequest("http://example.com", if_modified_since=http_date)
         self.assertEqual(
             request.headers, {"If-Modified-Since": format_timestamp(http_date)}
diff --git a/tornado/test/httputil_test.py b/tornado/test/httputil_test.py
index 8424491d87..aa9b6ee253 100644
--- a/tornado/test/httputil_test.py
+++ b/tornado/test/httputil_test.py
@@ -13,6 +13,7 @@
 from tornado.escape import utf8, native_str
 from tornado.log import gen_log
 from tornado.testing import ExpectLog
+from tornado.test.util import ignore_deprecation
 
 import copy
 import datetime
@@ -412,8 +413,29 @@ def test_time_tuple(self):
         self.assertEqual(9, len(tup))
         self.check(tup)
 
-    def test_datetime(self):
-        self.check(datetime.datetime.utcfromtimestamp(self.TIMESTAMP))
+    def test_utc_naive_datetime(self):
+        self.check(
+            datetime.datetime.fromtimestamp(
+                self.TIMESTAMP, datetime.timezone.utc
+            ).replace(tzinfo=None)
+        )
+
+    def test_utc_naive_datetime_deprecated(self):
+        with ignore_deprecation():
+            self.check(datetime.datetime.utcfromtimestamp(self.TIMESTAMP))
+
+    def test_utc_aware_datetime(self):
+        self.check(
+            datetime.datetime.fromtimestamp(self.TIMESTAMP, datetime.timezone.utc)
+        )
+
+    def test_other_aware_datetime(self):
+        # Other timezones are ignored; the timezone is always printed as GMT
+        self.check(
+            datetime.datetime.fromtimestamp(
+                self.TIMESTAMP, datetime.timezone(datetime.timedelta(hours=-4))
+            )
+        )
 
 
 # HTTPServerRequest is mainly tested incidentally to the server itself,
diff --git a/tornado/test/locale_test.py b/tornado/test/locale_test.py
index ee74cb05e8..a2e0872b8f 100644
--- a/tornado/test/locale_test.py
+++ b/tornado/test/locale_test.py
@@ -91,45 +91,55 @@ def test_format_date(self):
             locale.format_date(date, full_format=True), "April 28, 2013 at 6:35 pm"
         )
 
-        now = datetime.datetime.utcnow()
-
-        self.assertEqual(
-            locale.format_date(now - datetime.timedelta(seconds=2), full_format=False),
-            "2 seconds ago",
-        )
-        self.assertEqual(
-            locale.format_date(now - datetime.timedelta(minutes=2), full_format=False),
-            "2 minutes ago",
-        )
-        self.assertEqual(
-            locale.format_date(now - datetime.timedelta(hours=2), full_format=False),
-            "2 hours ago",
-        )
-
-        self.assertEqual(
-            locale.format_date(
-                now - datetime.timedelta(days=1), full_format=False, shorter=True
-            ),
-            "yesterday",
-        )
-
-        date = now - datetime.timedelta(days=2)
-        self.assertEqual(
-            locale.format_date(date, full_format=False, shorter=True),
-            locale._weekdays[date.weekday()],
-        )
-
-        date = now - datetime.timedelta(days=300)
-        self.assertEqual(
-            locale.format_date(date, full_format=False, shorter=True),
-            "%s %d" % (locale._months[date.month - 1], date.day),
-        )
-
-        date = now - datetime.timedelta(days=500)
-        self.assertEqual(
-            locale.format_date(date, full_format=False, shorter=True),
-            "%s %d, %d" % (locale._months[date.month - 1], date.day, date.year),
-        )
+        aware_dt = datetime.datetime.now(datetime.timezone.utc)
+        naive_dt = aware_dt.replace(tzinfo=None)
+        for name, now in {"aware": aware_dt, "naive": naive_dt}.items():
+            with self.subTest(dt=name):
+                self.assertEqual(
+                    locale.format_date(
+                        now - datetime.timedelta(seconds=2), full_format=False
+                    ),
+                    "2 seconds ago",
+                )
+                self.assertEqual(
+                    locale.format_date(
+                        now - datetime.timedelta(minutes=2), full_format=False
+                    ),
+                    "2 minutes ago",
+                )
+                self.assertEqual(
+                    locale.format_date(
+                        now - datetime.timedelta(hours=2), full_format=False
+                    ),
+                    "2 hours ago",
+                )
+
+                self.assertEqual(
+                    locale.format_date(
+                        now - datetime.timedelta(days=1),
+                        full_format=False,
+                        shorter=True,
+                    ),
+                    "yesterday",
+                )
+
+                date = now - datetime.timedelta(days=2)
+                self.assertEqual(
+                    locale.format_date(date, full_format=False, shorter=True),
+                    locale._weekdays[date.weekday()],
+                )
+
+                date = now - datetime.timedelta(days=300)
+                self.assertEqual(
+                    locale.format_date(date, full_format=False, shorter=True),
+                    "%s %d" % (locale._months[date.month - 1], date.day),
+                )
+
+                date = now - datetime.timedelta(days=500)
+                self.assertEqual(
+                    locale.format_date(date, full_format=False, shorter=True),
+                    "%s %d, %d" % (locale._months[date.month - 1], date.day, date.year),
+                )
 
     def test_friendly_number(self):
         locale = tornado.locale.get("en_US")
diff --git a/tornado/test/web_test.py b/tornado/test/web_test.py
index 56900d9a00..fb9c3417b9 100644
--- a/tornado/test/web_test.py
+++ b/tornado/test/web_test.py
@@ -404,10 +404,10 @@ def test_set_cookie_expires_days(self):
         match = re.match("foo=bar; expires=(?P<expires>.+); Path=/", header)
         assert match is not None
 
-        expires = datetime.datetime.utcnow() + datetime.timedelta(days=10)
-        parsed = email.utils.parsedate(match.groupdict()["expires"])
-        assert parsed is not None
-        header_expires = datetime.datetime(*parsed[:6])
+        expires = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
+            days=10
+        )
+        header_expires = email.utils.parsedate_to_datetime(match.groupdict()["expires"])
         self.assertTrue(abs((expires - header_expires).total_seconds()) < 10)
 
     def test_set_cookie_false_flags(self):
@@ -1697,11 +1697,10 @@ def get(self):
 
     def test_date_header(self):
         response = self.fetch("/")
-        parsed = email.utils.parsedate(response.headers["Date"])
-        assert parsed is not None
-        header_date = datetime.datetime(*parsed[:6])
+        header_date = email.utils.parsedate_to_datetime(response.headers["Date"])
         self.assertTrue(
-            header_date - datetime.datetime.utcnow() < datetime.timedelta(seconds=2)
+            header_date - datetime.datetime.now(datetime.timezone.utc)
+            < datetime.timedelta(seconds=2)
         )
 
 
@@ -3010,10 +3009,12 @@ def test_xsrf_httponly(self):
         match = re.match(".*; expires=(?P<expires>.+);.*", header)
         assert match is not None
 
-        expires = datetime.datetime.utcnow() + datetime.timedelta(days=2)
-        parsed = email.utils.parsedate(match.groupdict()["expires"])
-        assert parsed is not None
-        header_expires = datetime.datetime(*parsed[:6])
+        expires = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
+            days=2
+        )
+        header_expires = email.utils.parsedate_to_datetime(match.groupdict()["expires"])
+        if header_expires.tzinfo is None:
+            header_expires = header_expires.replace(tzinfo=datetime.timezone.utc)
         self.assertTrue(abs((expires - header_expires).total_seconds()) < 10)
 
 
diff --git a/tornado/web.py b/tornado/web.py
index 565140493e..439e02c47b 100644
--- a/tornado/web.py
+++ b/tornado/web.py
@@ -647,7 +647,9 @@ def set_cookie(
         if domain:
             morsel["domain"] = domain
         if expires_days is not None and not expires:
-            expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days)
+            expires = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
+                days=expires_days
+            )
         if expires:
             morsel["expires"] = httputil.format_timestamp(expires)
         if path:
@@ -698,7 +700,9 @@ def clear_cookie(self, name: str, **kwargs: Any) -> None:
                 raise TypeError(
                     f"clear_cookie() got an unexpected keyword argument '{excluded_arg}'"
                 )
-        expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
+        expires = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(
+            days=365
+        )
         self.set_cookie(name, value="", expires=expires, **kwargs)
 
     def clear_all_cookies(self, **kwargs: Any) -> None:
@@ -2812,12 +2816,12 @@ def should_return_304(self) -> bool:
         # content has not been modified
         ims_value = self.request.headers.get("If-Modified-Since")
         if ims_value is not None:
-            date_tuple = email.utils.parsedate(ims_value)
-            if date_tuple is not None:
-                if_since = datetime.datetime(*date_tuple[:6])
-                assert self.modified is not None
-                if if_since >= self.modified:
-                    return True
+            if_since = email.utils.parsedate_to_datetime(ims_value)
+            if if_since.tzinfo is None:
+                if_since = if_since.replace(tzinfo=datetime.timezone.utc)
+            assert self.modified is not None
+            if if_since >= self.modified:
+                return True
 
         return False
 
@@ -2981,6 +2985,10 @@ def get_modified_time(self) -> Optional[datetime.datetime]:
         object or None.
 
         .. versionadded:: 3.1
+
+        .. versionchanged:: 6.4
+           Now returns an aware datetime object instead of a naive one.
+           Subclasses that override this method may return either kind.
         """
         stat_result = self._stat()
         # NOTE: Historically, this used stat_result[stat.ST_MTIME],
@@ -2991,7 +2999,9 @@ def get_modified_time(self) -> Optional[datetime.datetime]:
         # consistency with the past (and because we have a unit test
         # that relies on this), we truncate the float here, although
         # I'm not sure that's the right thing to do.
-        modified = datetime.datetime.utcfromtimestamp(int(stat_result.st_mtime))
+        modified = datetime.datetime.fromtimestamp(
+            int(stat_result.st_mtime), datetime.timezone.utc
+        )
         return modified
 
     def get_content_type(self) -> str:
