pumpwood_communication.serializers

  1"""Miscellaneous to help with serializers in communication."""
  2import base64
  3import simplejson as json
  4import numpy as np
  5import pandas as pd
  6from typing import List, Union, Dict
  7from simplejson import JSONEncoder
  8from datetime import datetime
  9from datetime import date
 10from datetime import time
 11from pandas import Timestamp
 12from shapely.geometry.base import BaseGeometry
 13from shapely.geometry import mapping
 14from sqlalchemy_utils.types.choice import Choice
 15from pumpwood_communication.exceptions import (
 16    PumpWoodException, PumpWoodNotImplementedError)
 17
 18
 19class PumpWoodJSONEncoder(JSONEncoder):
 20    """PumpWood default serializer.
 21
 22    Treat not simple python types to facilitate at serialization of
 23    pandas, numpy, data, datetime and other data types.
 24    """
 25
 26    def default(self, obj):
 27        """Serialize complex objects."""
 28        # Return None if object is NaN
 29        if isinstance(obj, datetime):
 30            return obj.isoformat()
 31        if isinstance(obj, Timestamp):
 32            return obj.isoformat()
 33        if isinstance(obj, date):
 34            return obj.isoformat()
 35        if isinstance(obj, time):
 36            return obj.isoformat()
 37        if isinstance(obj, np.ndarray):
 38            return obj.tolist()
 39        if isinstance(obj, pd.DataFrame):
 40            return obj.to_dict('records')
 41        if isinstance(obj, pd.Series):
 42            obj.dtype == Timestamp
 43            return obj.tolist()
 44        if isinstance(obj, np.generic):
 45            return obj.item()
 46        if isinstance(obj, BaseGeometry):
 47            if obj.is_empty:
 48                return None
 49            else:
 50                return mapping(obj)
 51        if isinstance(obj, BaseGeometry):
 52            return mapping(obj)
 53        if isinstance(obj, Choice):
 54            return obj.code
 55        if isinstance(obj, set):
 56            return list(obj)
 57        else:
 58            raise TypeError(
 59                "Unserializable object {} of type {}".format(obj, type(obj)))
 60
 61
 62def pumpJsonDump(x: any, sort_keys: bool = True, indent: int = None): # NOQA
 63    """Dump a Json to python object.
 64
 65    Args:
 66        x (any):
 67            Object to be serialized using PumpWoodJSONEncoder encoder.
 68        sort_keys (bool):
 69            If json serialized data should have its keys sorted. This option
 70            makes serialization return of data reproductable.
 71        indent (int):
 72            Pass indent argument to simplejson dumps.
 73    """
 74    return json.dumps(
 75        x, cls=PumpWoodJSONEncoder, ignore_nan=True,
 76        sort_keys=sort_keys, indent=indent)
 77
 78
 79class CompositePkBase64Converter:
 80    """Convert composite primary keys in base64 dictionary."""
 81
 82    @staticmethod
 83    def dump(obj, primary_keys: Union[str, List[str], Dict[str, str]]
 84             ) -> Union[str, int]:
 85        """Convert primary keys and composite to a single value.
 86
 87        Treat cases when more than one column are used as primary keys,
 88        at this cases, a base64 used on url serialization of the dictionary
 89        is returned.
 90
 91        Args:
 92            obj:
 93                SQLAlchemy object.
 94            primary_keys (Union[str, List[str], Dict[str, str]):
 95                As string, a list or a dictionary leading to different
 96                behaviour.
 97                - **str:** It will return the value associated with object
 98                    attribute.
 99                - **List[str]:** If list has lenght equal to 1, it will have
100                    same behaviour as str. If greater than 1, it will be
101                    returned a base64 encoded dictionary with the keys at
102                    primary_keys.
103                - **Dict[str, str]:** Dictionary to map object fields to
104                    other keys. This is usefull when querying related fields
105                    by composite forenging keys to match original data fieds.
106
107        Returns:
108            If the primary key is unique, return the value of the primary
109            key, if is have more than one column as primary key, return
110            a dictionary of the primary keys encoded as base64 url safe.
111        """
112        if type(primary_keys) is str:
113            return getattr(obj, primary_keys)
114
115        elif type(primary_keys) is list:
116            if len(primary_keys) == 1:
117                return getattr(obj, primary_keys[0])
118            else:
119                # Will return a None value if all composite primary keys are
120                # None
121                is_all_none = False
122                composite_pk_dict = {}
123                for pk_col in primary_keys:
124                    temp_pk_value = getattr(obj, pk_col)
125                    is_all_none = is_all_none or (temp_pk_value is None)
126                    composite_pk_dict[pk_col] = temp_pk_value
127                if is_all_none:
128                    return None
129
130                # If not all primary keys are None, them serialize it and
131                # convert to a base64 dictionary to be used as PK
132                composite_pk_str = pumpJsonDump(composite_pk_dict)
133                return base64.urlsafe_b64encode(
134                    composite_pk_str.encode()).decode()
135
136        # Map object values to other, this is used when builds forenging
137        # key references and request related field using microservice.
138        elif type(primary_keys) is dict:
139            # Will return a None value if all composite primary keys are
140            # None
141            is_all_none = False
142            composite_pk_dict = {}
143            for key, value in primary_keys.items():
144                temp_pk_value = getattr(obj, key)
145                is_all_none = is_all_none or (temp_pk_value is None)
146                composite_pk_dict[value] = temp_pk_value
147            if is_all_none:
148                return None
149
150            # If not all primary keys are None, them serialize it and
151            # convert to a base64 dictionary to be used as PK. Using
152            # dictionary will map values before converting to base64
153            # dictionary
154            composite_pk_str = pumpJsonDump(composite_pk_dict)
155            base64_composite_pk = base64.urlsafe_b64encode(
156                composite_pk_str.encode()).decode()
157            return base64_composite_pk
158
159        # This will raise error if primary_keys type is not implemented
160        else:
161            msg = (
162                "CompositePkBase64Converter.dump argument primary_keys "
163                "is not a list of strings or a map dictionary. Type "
164                "[{arg_type}]").format(arg_type=type(primary_keys))
165            raise PumpWoodNotImplementedError(message=msg)
166
167    @staticmethod
168    def load(value: Union[str, int]) -> Union[int, dict]:
169        """Convert encoded primary keys to values.
170
171        If the primary key is a string, try to transform it to dictionary
172        decoding json base64 to a dictionary.
173
174        Args:
175            value:
176                Primary key value as an integer or as a base64
177                encoded json dictionary.
178
179        Return:
180            Return the primary key as integer if possible, or try to decoded
181            it to a dictionary from a base64 encoded json.
182        """
183        # Try to convert value to integer
184        try:
185            float_value = float(value)
186            if float_value.is_integer():
187                return int(float_value)
188            else:
189                msg = "[{value}] value is a float, but not integer."
190                raise PumpWoodException(msg, payload={"value": value})
191
192        # If not possible, try to decode a base64 JSON dictionary
193        except Exception as e1:
194            try:
195                return json.loads(base64.b64decode(value))
196            except Exception as e2:
197                msg = (
198                    "[{value}] value is not an integer and could no be "
199                    "decoded as a base64 encoded json dictionary. Value=")
200                raise PumpWoodException(
201                    message=msg, payload={
202                        "value": value,
203                        "exception_int": str(e1),
204                        "exception_base64": str(e2)})