feat: json_object_hook and serializer example (#294)

This adds a new feature, the ability to customize the json
deserialization into objects using a json.loads object hook.

it also updates documentation and includes an example of customizing
serialization and deserialization of python objects

#289

docs/serialization.rst

@@ -16,6 +16,7 @@ Complex data types
 ------------------
 Complex data types are dumped to json with json.dumps().
 
-Custom serialization is possible by overriding the serialize_partially and deserialize_partially methods in `AbstractModel <https://github.com/andrewthetechie/pydantic-aioredis/blob/main/pydantic_aioredis/abstract.py#L32>`_.
+Custom serialization is possible using `json_default <https://docs.python.org/3/library/json.html#:~:text=not%20None.-,If%20specified%2C%20default%20should%20be%20a%20function%20that%20gets%20called%20for%20objects%20that%20can%E2%80%99t%20otherwise%20be%20serialized.%20It%20should%20return%20a%20JSON%20encodable%20version%20of%20the%20object%20or%20raise%20a%20TypeError.%20If%20not%20specified%2C%20TypeError%20is%20raised.,-If%20sort_keys%20is>`_ and `json_object_hook <https://docs.python.org/3/library/json.html#:~:text=object_hook%20is%20an%20optional%20function%20that%20will%20be%20called%20with%20the%20result%20of%20any%20object%20literal%20decoded%20(a%20dict).%20The%20return%20value%20of%20object_hook%20will%20be%20used%20instead%20of%20the%20dict.%20This%20feature%20can%20be%20used%20to%20implement%20custom%20decoders%20(e.g.%20JSON%2DRPC%20class%20hinting).>`_.
 
-It is also possilbe to override json_default in AbstractModel. json_default is a callable used to convert any objects of a type json.dump cannot natively dump to string.
+These methods are part of the `abstract model <https://github.com/andrewthetechie/pydantic-aioredis/blob/main/pydantic_aioredis/abstract.py#L77>`_ and can be overridden in your
+model to dump custom objects to json and then back to objects. An example is available in `examples <https://github.com/andrewthetechie/pydantic-aioredis/tree/main/examples/serializer>`_

examples/serializer/Makefile

@@ -0,0 +1,5 @@
+start-redis: ## Runs a copy of redis in docker
+	docker run -it -d --rm --name pydantic-aioredis-example -p 6379:6379 -e REDIS_PASSWORD=password bitnami/redis || echo "$(REDIS_CONTAINER_NAME) is either running or failed"
+
+stop-redis: ## Stops the redis in docker
+	docker stop pydantic-aioredis-example

examples/serializer/README.md

@@ -0,0 +1,30 @@
+# asyncio_example
+
+This is a working example using python-aioredis with asyncio and a custom serializer for a python object BookCover.
+
+Book.json_default is used to serialize the BookCover object to a dictionary that json.dumps can dump to a string and store in redis.
+Book.json_object_hook can convert a dict from redis back to a BookCover object.
+
+# Requirements
+
+This example requires a running redis server. You can change the RedisConfig on line 28 in the example to match connecting to your running redis.
+
+For your ease of use, we've provided a Makefile in this directory that can start and stop a redis using docker.
+
+`make start-redis`
+
+`make stop-redis`
+
+The example is configured to connect to this dockerized redis automatically
+
+# Expected Output
+
+This is a working example. If you try to run it and find it broken, first check your local env. If you are unable to get the
+example running, please raise an Issue
+
+```bash
+python custom_serializer.py
+[Book(title='Great Expectations', author='Charles Dickens', published_on=datetime.date(1220, 4, 4), cover=<__main__.BookCover object at 0x10410c4c0>), Book(title='Jane Eyre', author='Charlotte Bronte', published_on=datetime.date(1225, 6, 4), cover=<__main__.BookCover object at 0x10410d4e0>), Book(title='Moby Dick', author='Herman Melville', published_on=datetime.date(1851, 10, 18), cover=<__main__.BookCover object at 0x10410d060>), Book(title='Oliver Twist', author='Charles Dickens', published_on=datetime.date(1215, 4, 4), cover=<__main__.BookCover object at 0x10410c760>), Book(title='Wuthering Heights', author='Emily Bronte', published_on=datetime.date(1600, 4, 4), cover=<__main__.BookCover object at 0x10410d690>)]
+[Book(title='Jane Eyre', author='Charlotte Bronte', published_on=datetime.date(1225, 6, 4), cover=<__main__.BookCover object at 0x10410cdc0>), Book(title='Oliver Twist', author='Charles Dickens', published_on=datetime.date(1215, 4, 4), cover=<__main__.BookCover object at 0x10410d7e0>)]
+[{'author': 'Charles Dickens', 'cover': <__main__.BookCover object at 0x10410d7b0>}, {'author': 'Charlotte Bronte', 'cover': <__main__.BookCover object at 0x10410d8d0>}, {'author': 'Herman Melville', 'cover': <__main__.BookCover object at 0x10410d840>}, {'author': 'Charles Dickens', 'cover': <__main__.BookCover object at 0x10410d960>}, {'author': 'Emily Bronte', 'cover': <__main__.BookCover object at 0x10410d900>}]
+```

examples/serializer/custom_serializer.py

@@ -0,0 +1,156 @@
+import asyncio
+import json
+from datetime import date
+from datetime import datetime
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+
+from pydantic_aioredis import Model
+from pydantic_aioredis import RedisConfig
+from pydantic_aioredis import Store
+from pydantic_aioredis.abstract import STR_DUMP_SHAPES
+
+
+class BookCover:
+    def __init__(self, cover_url: int, cover_size_x: int, cover_size_y: int):
+        self.cover_url = cover_url
+        self.cover_size_x = cover_size_x
+        self.cover_size_y = cover_size_y
+
+    @property
+    def area(self):
+        return self.cover_size_x * self.cover_size_y
+
+
+# Create models as you would create pydantic models i.e. using typings
+class Book(Model):
+    _primary_key_field: str = "title"
+    title: str
+    author: str
+    published_on: date
+    cover: BookCover
+
+    @classmethod
+    def json_default(cls, obj: Any) -> str:
+        """Since BookCover can't be directly json serialized, we have to write our own json_default to serialize it methods to handle it."""
+        if isinstance(obj, BookCover):
+            return {
+                "__BookCover__": True,
+                "cover_url": obj.cover_url,
+                "cover_size_x": obj.cover_size_x,
+                "cover_size_y": obj.cover_size_y,
+            }
+
+        return super().json_default(obj)
+
+    @classmethod
+    def json_object_hook(cls, obj: dict):
+        """Since we're serializing BookCovers above, we need to write an object hook to turn them back into an Object"""
+        if obj.get("__BookCover__", False):
+            return BookCover(
+                cover_url=obj["cover_url"],
+                cover_size_x=obj["cover_size_x"],
+                cover_size_y=obj["cover_size_y"],
+            )
+        super().json_object_hook(obj)
+
+
+# Redisconfig. Change this configuration to match your redis server
+redis_config = RedisConfig(
+    db=5, host="localhost", password="password", ssl=False, port=6379
+)
+
+
+# Create the store and register your models
+store = Store(name="some_name", redis_config=redis_config, life_span_in_seconds=3600)
+store.register_model(Book)
+
+
+# Sample books. You can create as many as you wish anywhere in the code
+books = [
+    Book(
+        title="Oliver Twist",
+        author="Charles Dickens",
+        published_on=date(year=1215, month=4, day=4),
+        cover=BookCover(
+            "https://images-na.ssl-images-amazon.com/images/I/51SmEM7LUGL._SX342_SY445_QL70_FMwebp_.jpg",
+            333,
+            499,
+        ),
+    ),
+    Book(
+        title="Great Expectations",
+        author="Charles Dickens",
+        published_on=date(year=1220, month=4, day=4),
+        cover=BookCover(
+            "https://images-na.ssl-images-amazon.com/images/I/51i715XqsYL._SX311_BO1,204,203,200_.jpg",
+            333,
+            499,
+        ),
+    ),
+    Book(
+        title="Jane Eyre",
+        author="Charlotte Bronte",
+        published_on=date(year=1225, month=6, day=4),
+        cover=BookCover(
+            "https://images-na.ssl-images-amazon.com/images/I/41saarVx+GL._SX324_BO1,204,203,200_.jpg",
+            333,
+            499,
+        ),
+    ),
+    Book(
+        title="Wuthering Heights",
+        author="Emily Bronte",
+        published_on=date(year=1600, month=4, day=4),
+        cover=BookCover(
+            "https://images-na.ssl-images-amazon.com/images/I/51ZKox7zBKL._SX338_BO1,204,203,200_.jpg",
+            333,
+            499,
+        ),
+    ),
+]
+
+
+async def work_with_orm():
+    # Insert them into redis
+    await Book.insert(books)
+
+    # Select all books to view them. A list of Model instances will be returned
+    all_books = await Book.select()
+    print(all_books)  # Will print [Book(title="Oliver Twist", author="Charles Dickens",
+    # published_on=date(year=1215, month=4, day=4), in_stock=False), Book(...]
+
+    # Or select some of the books
+    some_books = await Book.select(ids=["Oliver Twist", "Jane Eyre"])
+    print(some_books)  # Will return only those two books
+
+    # Or select some of the columns. THIS RETURNS DICTIONARIES not MODEL Instances
+    # The Dictionaries have values in string form so you might need to do some extra work
+    books_with_few_fields = await Book.select(columns=["author", "cover"])
+    print(
+        books_with_few_fields
+    )  # Will print [{"author": "'Charles Dickens", "covker": Cover},...]
+
+    # When _auto_sync = True (default), updating any attribute will update that field in Redis too
+    this_book = Book(
+        title="Moby Dick",
+        author="Herman Melvill",
+        published_on=date(year=1851, month=10, day=18),
+        cover=BookCover(
+            "https://m.media-amazon.com/images/I/411a8Moy1mL._SY346_.jpg", 333, 499
+        ),
+    )
+    await Book.insert(this_book)
+    # oops, there was a typo. Fix it
+    this_book.author = "Herman Melville"
+    this_book_from_redis = await Book.select(ids=["Moby Dick"])
+    assert this_book_from_redis[0].author == "Herman Melville"
+
+    # If you have _auto_save set to false on a model, you have to await .save() to update a model in tedis
+    await this_book.save()
+
+
+if __name__ == "__main__":
+    asyncio.run(work_with_orm())

pydantic_aioredis/abstract.py

@@ -73,8 +73,13 @@ class _AbstractModel(BaseModel):
     _table_name: Optional[str] = None
     _auto_sync: bool = True
 
-    @staticmethod
-    def json_default(obj: Any) -> str:
+    @classmethod
+    def json_object_hook(cls, obj: dict):
+        """Can be overridden to handle custom json -> object"""
+        return obj
+
+    @classmethod
+    def json_default(cls, obj: Any) -> str:
         """
         JSON serializer for objects not serializable by default json library
         Currently handles: datetimes -> obj.isoformat, ipaddress and ipnetwork -> str
@@ -127,9 +132,9 @@ def deserialize_partially(cls, data: Dict[bytes, Any]):
             if field not in columns:
                 continue
             if cls.__fields__[field].type_ not in [str, float, int]:
-                data[field] = json.loads(data[field])
+                data[field] = json.loads(data[field], object_hook=cls.json_object_hook)
             if getattr(cls.__fields__[field], "shape", None) in JSON_DUMP_SHAPES:
-                data[field] = json.loads(data[field])
+                data[field] = json.loads(data[field], object_hook=cls.json_object_hook)
             if getattr(cls.__fields__[field], "allow_none", False):
                 if data[field] == "None":
                     data[field] = None