Fix pickling and add docs

Author: ddelange

docs/backends/amazon-S3.rst

@@ -259,13 +259,25 @@ Settings
     Setting this overrides the settings for ``addressing_style``, ``signature_version`` and
     ``proxies``. Include them as arguments to your ``botocore.config.Config`` class if you need them.
 
+``client_ttl`` or ``AWS_S3_CLIENT_TTL``
+
+  Default: ``3600``
+
+  The amount of seconds to store a boto3 client resource in an S3Storage instance's time-to-live cache.
+
+  .. note::
+
+    Long-lived boto3 clients have a known `memory leak`_, which is why the client is
+    periodically recreated to avoid excessive memory consumption.
+
 .. _AWS Signature Version 4: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
 .. _S3 region list: https://docs.aws.amazon.com/general/latest/gr/s3.html#s3_region
 .. _list of canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 .. _Boto3 docs for uploading files: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.put_object
 .. _Boto3 docs for TransferConfig: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.TransferConfig
 .. _ManifestStaticFilesStorage: https://docs.djangoproject.com/en/3.1/ref/contrib/staticfiles/#manifeststaticfilesstorage
 .. _Botocore docs: https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html#botocore.config.Config
+.. _memory leak: https://github.com/boto/boto3/issues/1670
 
 .. _cloudfront-signed-url-header:
 

pyproject.toml

@@ -50,6 +50,8 @@ azure = [
 ]
 boto3 = [
   "boto3>=1.4.4",
+  "cachetools>=5.0.0",
+  "backports.cached_property; python_version<='3.7'"
 ]
 dropbox = [
   "dropbox>=7.2.1",
@@ -63,6 +65,7 @@ libcloud = [
 s3 = [
   "boto3>=1.4.4",
   "cachetools>=5.0.0",
+  "backports.cached_property; python_version<='3.7'"
 ]
 sftp = [
   "paramiko>=1.15",

storages/backends/s3.py

@@ -36,6 +36,15 @@
     msg = "Could not import cachetools. Did you run 'pip install django-storages[s3]'?"
     raise ImproperlyConfigured(msg) from e
 
+try:
+    from functools import cached_property
+except ImportError:  # python_version<='3.7'
+    try:
+        from backports.cached_property import cached_property
+    except (ImportError, ModuleNotFoundError) as e:
+        msg = "Could not import backports.cached_property. Did you run 'pip install django-storages[s3]'?"  # noqa: E501
+        raise ImproperlyConfigured(msg) from e
+
 try:
     import boto3.session
     import botocore
@@ -379,10 +388,6 @@ def __init__(self, **settings):
             else:
                 self.cloudfront_signer = None
 
-        self._ttl_cache = cachetools.cached(
-            cache=cachetools.TTLCache(maxsize=2, ttl=3600), lock=threading.Lock()
-        )
-
     def get_cloudfront_signer(self, key_id, key):
         cache_key = f"{key_id}:{key}"
         if cache_key not in self.__class__._signers:
@@ -451,6 +456,7 @@ def get_default_settings(self):
             "use_threads": setting("AWS_S3_USE_THREADS", True),
             "transfer_config": setting("AWS_S3_TRANSFER_CONFIG", None),
             "client_config": setting("AWS_S3_CLIENT_CONFIG", None),
+            "client_ttl": setting("AWS_S3_CLIENT_TTL", 3600),
         }
 
     def __getstate__(self):
@@ -465,23 +471,27 @@ def __setstate__(self, state):
     def connection(self):
         """
         Get the (cached) thread-safe boto3 s3 resource.
-
-        This function has a 1 hour time to live cache for the boto3 resource.
-        We want to avoid storing a resource for too long to avoid their memory leak
-        ref https://github.com/boto/boto3/issues/1670.
         """
         return self._ttl_cache(self._create_connection)()
 
     @property
     def unsigned_connection(self):
         """
         Get the (cached) thread-safe boto3 s3 resource (unsigned).
+        """
+        return self._ttl_cache(self._create_connection)(unsigned=True)
 
-        This function has a 1 hour time to live cache for the boto3 resource.
+    @cached_property
+    def _ttl_cache(self):
+        """
+        This time to live cache is used to periodically recreate boto3 clients.
         We want to avoid storing a resource for too long to avoid their memory leak
         ref https://github.com/boto/boto3/issues/1670.
         """
-        return self._ttl_cache(self._create_connection)(unsigned=True)
+        return cachetools.cached(
+            cache=cachetools.TTLCache(maxsize=2, ttl=self.client_ttl),
+            lock=threading.Lock(),
+        )
 
     def _create_connection(self, *, unsigned=False):
         """

tests/test_s3.py

@@ -75,7 +75,9 @@ def test_pickle(self):
         _ = storage.connection
         _ = storage.bucket
         p = pickle.dumps(storage)
-        pickle.loads(p)
+        new_storage = pickle.loads(p)
+        _ = new_storage.connection
+        _ = storage.bucket
 
     def test_storage_url_slashes(self):
         """