pumpwood_communication.microservice_abc.simple

 21class ABCSimpleBatchMicroservice(ABC, PumpWoodMicroServiceBase):
 22    """Abstract class for batch end-points."""
 23
 24    @staticmethod
 25    def _build_aggregate_url(model_class: str):
 26        return "rest/%s/aggregate/" % (model_class.lower(),)
 27
 28    def aggregate(self, model_class: str, group_by: List[str] | str, agg: dict,
 29                  filter_dict: None | dict = None, exclude_dict: dict = None,
 30                  order_by: List[str] = None, auth_header: dict = None,
 31                  limit: int = None, as_dataframe: bool = False
 32                  ) -> pd.DataFrame:
 33        """Save a list of objects with one request.
 34
 35        Avaiable aggregate functions:
 36            - **sum:** Sum.
 37            - **mean:** Mean.
 38            - **count:** Count.
 39            - **min:** Minimum.
 40            - **max:** Maximum.
 41            - **stddev_pop:** Population standard deviation.
 42            - **stddev_samp:** Sample standard deviation.
 43            - **var_pop:** Population variance.
 44            - **var_samp:** Sample variance.
 45            - **std:** Sample standard deviation.
 46            - **var:** Sample variance.
 47
 48        Args:
 49            model_class (str):
 50                Model class of the end-point that will be aggregated.
 51            group_by (List[str] | str):
 52                List of the fields that will be used on aggregation as
 53                group by. If a string is passed as argument, it will be
 54                considered the sigle group_by column.
 55            agg (dict):
 56                A dictionary with dictionary items as `field` and `function`
 57                specifing the field that will be aggregated using a function.
 58
 59                The dictinary keys will be used to return the results as
 60                columns.
 61            filter_dict (dict):
 62                Filter that will be applied before the aggregation.
 63            exclude_dict (dict):
 64                Exclude clause that will be applied before the aggregation.
 65            order_by (list):
 66                Ordenation acording to grouping elements. It can be used
 67                fields created as keys of agg dictinary.
 68            auth_header (dict):
 69                Authentication header used to impersonation of user.
 70            limit (int):
 71                Limit number of returned row at aggregation query.
 72            as_dataframe (bool):
 73                If result should be returned as a dataframe. The columns
 74                will be set to match the group_by and agg arguments, this
 75                way empty dataframe with respect the columns.
 76
 77        Returns:
 78            Return a DataFrame with aggregation results.
 79
 80        Example:
 81            ```
 82            microservice.aggregate(
 83                model_class="ToLoadCalendar",
 84                group_by=["calendar_id"],
 85                agg={
 86                    "n": {"field": "id", "function": "count"},
 87                    "mean": {"field": "value", "function": "mean"
 88                }})
 89            ```
 90        """
 91        filter_dict = {} if filter_dict is None else filter_dict
 92        exclude_dict = {} if exclude_dict is None else exclude_dict
 93        order_by = [] if order_by is None else order_by
 94
 95        # If group_by is a string, convert to a list with this string
 96        group_by = [group_by] if isinstance(group_by, str) else group_by
 97        if not isinstance(group_by, (list, tuple, set)):
 98            error_msg = "Argument `group_by` must be list, tuple, set or str."
 99            raise TypeError(error_msg)
100        if not isinstance(agg, (dict)):
101            error_msg = "Argument `agg` must be dict."
102            raise TypeError(error_msg)
103
104        url_str = self._build_aggregate_url(model_class=model_class)
105        data = {
106            'agg': agg, 'group_by': group_by, 'filter_dict': filter_dict,
107            'exclude_dict': exclude_dict, 'order_by': order_by,
108            'limit': limit}
109        return_data = self.request_post(
110            url=url_str, data=data, auth_header=auth_header)
111        if not as_dataframe:
112            return return_data
113        else:
114            # Return the results as a dataframe using the group_by columns
115            # and the columns created at aggregation to ensure that
116            # even empty results would have the correct columns
117            return_columns = group_by + list(agg.keys())
118            return pd.DataFrame(return_data, columns=return_columns)
119
120    @staticmethod
121    def _build_pivot_url(model_class):
122        return "rest/%s/pivot/" % (model_class.lower(), )
123
124    def pivot(self, model_class: str, columns: None | List[str] = None,
125              format: str = 'list', filter_dict: dict = None,
126              exclude_dict: dict = None, order_by: List[str] = None,
127              variables: List[str] = None, fields: List[str] = None,
128              show_deleted: bool = False,
129              add_pk_column: bool = False, auth_header: dict = None,
130              as_dataframe: bool = False
131              ) -> Union[List[dict], Dict[str, list], pd.DataFrame]:
132        """Pivot object data acording to columns specified.
133
134        Pivoting per-se is not usually used, beeing the name of the function
135        a legacy. Normality data transformation is done at the client level.
136
137        Args:
138            model_class (str):
139                Model class to check search parameters.
140            columns (List[str]):
141                List of fields to be used as columns when pivoting the data.
142            format (str):
143                Format to be used to convert pandas.DataFrame to
144                dictionary, must be in ['dict','list','series',
145                'split', 'records','index'].
146            filter_dict (dict):
147                Same as list function.
148            exclude_dict (dict):
149                Same as list function.
150            order_by (List[str]):
151                 Same as list function.
152            fields (List[str]):
153                List of the fields to be returned, if None, the default
154                variables will be returned. Same as fields on list functions.
155            variables (List[str]):
156                List of the fields to be returned, if None, the default
157                variables will be returned. Same as fields on list functions.
158                **DEPRECTED** use fields.
159            show_deleted (bool):
160                Fields with deleted column will have objects with deleted=True
161                omited from results. show_deleted=True will return this
162                information.
163            add_pk_column (bool):
164                If add pk values of the objects at pivot results. Adding
165                pk key on pivot end-points won't be possible to pivot since
166                pk is unique for each entry.
167            auth_header (dict):
168                Auth header to substitute the microservice original
169                at the request (user impersonation).
170            as_dataframe (bool):
171                If results should be returned as a dataframe.
172
173        Returns:
174            Return a list or a dictinary depending on the format set on
175            format parameter.
176
177        Raises:
178            PumpWoodException:
179                'Columns must be a list of elements.'. Indicates that the list
180                argument was not a list.
181            PumpWoodException:
182                'Column chosen as pivot is not at model variables'. Indicates
183                that columns that were set to pivot are not present on model
184                variables.
185            PumpWoodException:
186                "Format must be in ['dict','list','series','split',
187                'records','index']". Indicates that format set as paramenter
188                is not implemented.
189            PumpWoodException:
190                "Can not add pk column and pivot information". If
191                add_pk_column is True (results will have the pk column), it is
192                not possible to pivot the information (pk is an unique value
193                for each object, there is no reason to pivot it).
194            PumpWoodException:
195                "'value' column not at melted data, it is not possible
196                to pivot dataframe.". Indicates that data does not have a value
197                column, it must have it to populate pivoted table.
198        """
199        # Deprect variables argument
200        if variables is not None:
201            msg = (
202                "Deprecation Warning: pivot variable is deprected, "
203                "you should use fields instead")
204            logger.warning(msg)
205        fields = fields if fields is not None else variables
206
207        filter_dict = {} if filter_dict is None else filter_dict
208        exclude_dict = {} if exclude_dict is None else exclude_dict
209        order_by = [] if order_by is None else order_by
210        columns = [] if columns is None else columns
211
212        url_str = self._build_pivot_url(model_class)
213        post_data = {
214            'columns': columns, 'format': format,
215            'filter_dict': filter_dict, 'exclude_dict': exclude_dict,
216            'order_by': order_by, "variables": variables,
217            "show_deleted": show_deleted, "add_pk_column": add_pk_column}
218        pivot_results = self.request_post(
219            url=url_str, data=post_data, auth_header=auth_header)
220
221        if not add_pk_column:
222            if as_dataframe:
223                # If passed, set the fields as columns to ensure empty
224                # dataframes to have the correct fields
225                return pd.DataFrame(pivot_results, columns=fields)
226            else:
227                return pivot_results
228        else:
229            # If passed, set the fields as columns to ensure empty
230            # dataframes to have the correct fields
231            pd_pivot_results = pd.DataFrame(pivot_results, columns=fields)
232            if len(pd_pivot_results) != 0:
233                fill_options = self.fill_options(
234                    model_class=model_class, auth_header=auth_header)
235                primary_keys_data = fill_options["pk"]
236                # Allow legacy pk definition and actual that uses extra_info
237                primary_keys = primary_keys_data\
238                    .get('extra_info', {})\
239                    .get('columns', primary_keys_data['column'])
240                pd_pivot_results["pk"] = pd_pivot_results[primary_keys].apply(
241                    CompositePkBase64Converter.dump,
242                    primary_keys=primary_keys, axis=1)
243            if as_dataframe:
244                return pd_pivot_results
245            else:
246                return pd_pivot_results.to_dict(format)
247
248    def _flat_list_by_chunks_helper(self, args):
249        try:
250            # Unpacking arguments
251            model_class = args["model_class"]
252            filter_dict = args["filter_dict"]
253            exclude_dict = args["exclude_dict"]
254            fields = args["fields"]
255            show_deleted = args["show_deleted"]
256            auth_header = args["auth_header"]
257            chunk_size = args["chunk_size"]
258
259            temp_filter_dict = copy.deepcopy(filter_dict)
260            url_str = self._build_pivot_url(model_class)
261            max_pk = 0
262
263            # Fetch data until an empty result is returned
264            list_dataframes = []
265            while True:
266                sys.stdout.write(".")
267                sys.stdout.flush()
268                temp_filter_dict["id__gt"] = max_pk
269                post_data = {
270                    'format': 'list',
271                    'filter_dict': temp_filter_dict,
272                    'exclude_dict': exclude_dict,
273                    'order_by': ["id"], "variables": fields,
274                    "show_deleted": show_deleted,
275                    "limit": chunk_size,
276                    "add_pk_column": True}
277                temp_dateframe = pd.DataFrame(self.request_post(
278                    url=url_str, data=post_data, auth_header=auth_header))
279
280                # Break if results are less than chunk size, so no more results
281                # are avaiable
282                if len(temp_dateframe) < chunk_size:
283                    list_dataframes.append(temp_dateframe)
284                    break
285
286                max_pk = int(temp_dateframe["id"].max())
287                list_dataframes.append(temp_dateframe)
288
289            if len(list_dataframes) == 0:
290                return pd.DataFrame()
291            else:
292                return pd.concat(list_dataframes)
293        except Exception as e:
294            raise Exception("Exception at flat_list_by_chunks:", str(e))
295
296    def flat_list_by_chunks(self, model_class: str, filter_dict: dict = None,
297                            exclude_dict: dict = None,
298                            fields: List[str] = None,
299                            show_deleted: bool = False,
300                            auth_header: dict = None,
301                            chunk_size: int = 1000000,
302                            n_parallel: int = None,
303                            create_composite_pk: bool = False,
304                            start_date: str = None,
305                            end_date: str = None,
306                            time_column: str = 'time') -> pd.DataFrame:
307        """Incrementally fetch data from pivot end-point.
308
309        Fetch data from pivot end-point paginating by id of chunk_size length.
310
311        If table is partitioned it will split the query acording to partition
312        to facilitate query at the database.
313
314        If start_date and end_date are set, also breaks the query by month
315        retrieving each month data in parallel.
316
317        Args:
318            model_class (str):
319                Model class to be pivoted.
320            filter_dict (dict):
321                Dictionary to to be used in objects.filter argument
322                (Same as list end-point).
323            exclude_dict (dict):
324                Dictionary to to be used in objects.exclude argument
325                (Same as list end-point).
326            fields (List[str] | None):
327                List of the variables to be returned,
328                if None, the default variables will be returned.
329                If fields is set, dataframe will return that columns
330                even if data is empty.
331            start_date (datetime | str):
332                Set a begin date for the query. If begin and end date are
333                set, query will be splited with chucks by month that will be
334                requested in parallel.
335            end_date (datetime | str):
336                Set a end date for the query. If begin and end date are
337                set, query will be splited with chucks by month that will be
338                requested in parallel.
339            show_deleted (bool):
340                If deleted data should be returned.
341            auth_header (dict):
342                Auth header to substitute the microservice original
343                at the request (user impersonation).
344            chunk_size (int):
345                Limit of data to fetch per call.
346            n_parallel (int):
347                Number of parallel process to perform.
348            create_composite_pk (bool):
349                If true and table has a composite pk, it will create pk
350                value based on the hash on the json serialized dictionary
351                of the components of the primary key.
352            time_column (str):
353                Column that will be considered on the partitioning of data
354                fetch.
355
356        Returns:
357            Returns a dataframe with all information fetched.
358
359        Raises:
360            No particular raise.
361        """
362        # Set empty dictionary for dictionary arguments
363        filter_dict = {} if filter_dict is None else filter_dict
364        exclude_dict = {} if exclude_dict is None else exclude_dict
365        original_fields = None if fields is None else fields
366
367        if n_parallel is None:
368            n_parallel = PUMPWOOD_COMUNICATION__N_PARALLEL
369
370        fill_options = self.fill_options(
371            model_class=model_class, auth_header=auth_header)
372
373        # Retrieve PK infomration about primary_key and partitions
374        primary_keys = fill_options["pk"]["extra_info"]['columns']
375        # partition = fill_options["pk"]["extra_info"]['partition']
376
377        # Add all primary keys fields to query if create_composite_pk is
378        # set true
379        if create_composite_pk and fields is not None:
380            fields = list(set(fields) | set(primary_keys))
381
382        # Create a list of month and include start and end dates if not at
383        # the beginning of a month
384        month_sequence = None
385        if (start_date is not None) and (end_date is not None):
386            month_sequence = AuxFlatListByChunks.build_month_partitions(
387                start_date=start_date,
388                end_date=end_date)
389        elif (start_date is not None) or (end_date is not None):
390            # To create the partitions is necessary to have both start and end
391            # date.
392            msg = (
393                "To break query in chunks using start_date and end_date "
394                "both must be set.\n- start_date: {start_date}\n"
395                "- end_date: {end_date}")
396            raise PumpWoodException(
397                message=msg, payload={
398                    "start_date": start_date,
399                    "end_date": end_date})
400
401        pool_arguments = AuxFlatListByChunks.build_query_partitions(
402            model_class=model_class, time_partitions=month_sequence,
403            filter_dict=filter_dict, exclude_dict=exclude_dict, fields=fields,
404            time_column=time_column, show_deleted=show_deleted,
405            auth_header=auth_header, chunk_size=chunk_size)
406
407        # Perform parallel calls to backend each chucked by chunk_size
408        print("## Starting parallel flat list: %s" % len(pool_arguments))
409        resp_df = None
410        try:
411            with Pool(n_parallel) as p:
412                results = p.map(
413                    self._flat_list_by_chunks_helper,
414                    pool_arguments)
415            if len(results) != 0:
416                resp_df = pd.concat(results)\
417                    .reset_index(drop=True)
418            else:
419                resp_df = pd.DataFrame(columns=fields)
420        except Exception as e:
421            PumpWoodException(message=str(e))
422        print("\n## Finished parallel flat list: %s" % len(pool_arguments))
423
424        # Add the primary key as a column to the dataframe
425        resp_df = AuxFlatListByChunks.add_pk_column(
426            create_composite_pk=create_composite_pk,
427            primary_key_list=primary_keys,
428            data=resp_df)
429
430        # Limit the return fields
431        if original_fields is not None:
432            if create_composite_pk:
433                return pd.DataFrame(
434                    resp_df, columns=['pk'] + original_fields)
435            else:
436                return pd.DataFrame(
437                    resp_df, columns=original_fields)
438        else:
439            return resp_df
440
441    def get_pks_from_unique_field(self, model_class: str, field: str,
442                                  values: List[Any]) -> pd.DataFrame:
443        """Get pk using unique fields values.
444
445        Use unique field values to retrieve pk of the objects. This end-point
446        is usefull for retrieving pks of the objects associated with unique
447        fields such as `description` (unique on most model of pumpwood).
448
449        ```python
450        # Using description to fetch pks from objects
451        data: pd.DataFrame = [data with unique description but without pk]
452        data['attribute_id'] = microservice.get_pks_from_unique_field(
453            model_class="DescriptionAttribute",
454            field="description", values=data['attribute'])['pk']
455
456        # Using a dimension key to fetch pk of the objects, dimension
457        # key must be unique
458        data['georea_id'] = microservice.get_pks_from_unique_field(
459            model_class="DescriptionGeoarea", field="dimension->city",
460            values=data['city'])['pk']
461        ```
462
463        Args:
464            model_class:
465                Model class of the objects.
466            field:
467                Unique field to fetch pk. It is possible to use dimension keys
468                as unique field, for that use `dimension->[key]` notation.
469            values:
470                List of the unique fields used to fetch primary keys.
471
472        Return:
473            Return a dataframe in same order as values with columns:
474            - **pk**: Correspondent primary key of the unique value.
475            - **[field]**: Column with same name of field argument,
476                correspondent to pk.
477
478        Raises:
479            PumpWoodQueryException:
480                Raises if field is not found on the model and it is note
481                associated with a dimension tag.
482            PumpWoodQueryException:
483                Raises if `field` does not have a unique restriction on
484                database. Dimension keys does not check for uniqueness on
485                database, be carefull not to duplicate the lines.
486        """
487        is_dimension_tag = 'dimensions->' in field
488        if not is_dimension_tag:
489            fill_options = self.fill_options(model_class=model_class)
490            field_details = fill_options.get(field)
491            if field_details is None:
492                msg = (
493                    "Field is not a dimension tag and not found on model "
494                    "fields. Field [{field}]")
495                raise PumpWoodQueryException(
496                    message=msg, payload={"field": field})
497
498            is_unique_field = field_details.get("unique", False)
499            if not is_unique_field:
500                msg = "Field [{field}] to get pk from is not unique"
501                raise PumpWoodQueryException(
502                    message=msg, payload={"field": field})
503
504        filter_dict = {field + "__in": list(set(values))}
505        pk_map = None
506        if not is_dimension_tag:
507            list_results = pd.DataFrame(self.list_without_pag(
508                model_class=model_class, filter_dict=filter_dict,
509                fields=["pk", field]), columns=["pk", field])
510            pk_map = list_results.set_index(field)["pk"]
511
512        # If is dimension tag, fetch dimension and unpack it
513        else:
514            dimension_tag = field.split("->")[1]
515            list_results = pd.DataFrame(self.list_without_pag(
516                model_class=model_class, filter_dict=filter_dict,
517                fields=["pk", "dimensions"]))
518            pk_map = {}
519            if len(list_results) != 0:
520                pk_map = list_results\
521                    .pipe(unpack_dict_columns, columns=["dimensions"])\
522                    .set_index(dimension_tag)["pk"]
523
524        values_series = pd.Series(values)
525        return pd.DataFrame({
526            "pk": values_series.map(pk_map).to_numpy(),
527            field: values_series
528        })

 16class ABCSimpleRetriveMicroservice(ABC, PumpWoodMicroServiceBase):
 17    """Abstract class for parallel calls at Pumpwood end-points."""
 18
 19    @staticmethod
 20    def _build_list_one_url(model_class, pk):
 21        return "rest/%s/retrieve/%s/" % (model_class.lower(), pk)
 22
 23    def list_one(self, model_class: str, pk: int, fields: list = None,
 24                 default_fields: bool = True, foreign_key_fields: bool = False,
 25                 related_fields: bool = False, auth_header: dict = None,
 26                 use_disk_cache: bool = False, use_app_cache: bool = False,
 27                 disk_cache_expire: int = None,
 28                 base_filter_skip: list = None) -> dict:
 29        """Retrieve an object using list serializer (simple).
 30
 31        **# DEPRECTED #** It is the same as retrieve using
 32        `default_fields: bool = True`, if possible migrate to retrieve
 33        function.
 34
 35        Args:
 36            model_class:
 37                Model class of the end-point
 38            pk:
 39                Object pk
 40            auth_header:
 41                Auth header to substitute the microservice original
 42                at the request (user impersonation).
 43            fields:
 44                Set the fields to be returned by the list end-point.
 45            default_fields:
 46                Boolean, if true and fields arguments None will return the
 47                default fields set for list by the backend.
 48            foreign_key_fields:
 49                Return forenging key objects. It will return the fk
 50                corresponding object. Ex: `created_by_id` reference to
 51                a user `model_class` the correspondent to User will be
 52                returned at `created_by`.
 53            related_fields:
 54                Return related fields objects. Related field objects are
 55                objects that have a forenging key associated with this
 56                model_class, results will be returned as a list of
 57                dictionaries usually in a field with `_set` at end.
 58                Returning related_fields consume backend resorces, use
 59                carefully.
 60            use_disk_cache (bool):
 61                If set true, get request will use local cache to reduce
 62                the requests to the backend.
 63            use_app_cache (bool):
 64                If True, the GET request will use the cache of the application.
 65                Defaults to False.
 66            disk_cache_expire (int):
 67                Time in seconds to expire the cache, it None it will
 68                use de default set be PumpwoodCache.
 69            base_filter_skip (list[str]):
 70                List of base query filter to be skiped, it is necessary to
 71                be superuser to skip base query filters.
 72
 73        Returns:
 74            Return object with the correspondent pk.
 75
 76        Raises:
 77            PumpWoodObjectDoesNotExist:
 78                If pk not found on database.
 79        """
 80        base_filter_skip = (
 81            [] if base_filter_skip is None else base_filter_skip)
 82        url_str = self._build_list_one_url(model_class, pk)
 83        return self.request_get(
 84            url=url_str, parameters={
 85                "fields": fields, "default_fields": default_fields,
 86                "foreign_key_fields": foreign_key_fields,
 87                "related_fields": related_fields,
 88                "base_filter_skip": base_filter_skip,
 89                "use_cache": use_app_cache},
 90            auth_header=auth_header, use_disk_cache=use_disk_cache,
 91            disk_cache_expire=disk_cache_expire)
 92
 93    @staticmethod
 94    def _build_retrieve_url(model_class: str, pk: int):
 95        return "rest/%s/retrieve/%s/" % (model_class.lower(), pk)
 96
 97    def retrieve(self, model_class: str, pk: int | str | dict,
 98                 default_fields: bool = False,
 99                 foreign_key_fields: bool = False,
100                 related_fields: bool = False,
101                 fields: list = None,
102                 auth_header: dict = None,
103                 use_disk_cache: bool = False,
104                 use_app_cache: bool = False,
105                 disk_cache_expire: int = None,
106                 base_filter_skip: list = None) -> dict:
107        """Retrieve an object from PumpWood.
108
109        Function to get an object serialized by the retrieve endpoint (more
110        detailed data). It will fetch information for a single object
111        based on the primary key, which may be a simple ID, a composite key
112        passed as a dictionary, or a base64 URL-safe string.
113
114        It is also possible to retrieve single objects using unique fields,
115        such as codes or multiple column uniqueness constraints. This can
116        be done by passing the argument as a base64 string or a dictionary
117        containing the filtering clauses.
118
119        Example:
120            ```python
121            microservice.retrieve(
122                model_class="ModelClassWithUniqueCode",
123                pk={'code': 'unique code for object'})
124
125            microservice.retrieve(
126                model_class="ModelClassWithCompositeUniqueConstraint",
127                pk={'time': '2026-01-01', 'attribute_id': 1})
128            ```
129
130        Args:
131            model_class (str):
132                Model class of the endpoint.
133            pk (Union[int, str, dict]):
134                The primary key or unique identifier for the object.
135            auth_header (dict, optional):
136                Authentication header to substitute the microservice's original
137                credentials (used for user impersonation). Defaults to None.
138            fields (list, optional):
139                Set of fields to be returned by the endpoint.
140            default_fields (bool):
141                If True and 'fields' is None, will return the default fields
142                defined by the backend. Defaults to False.
143            foreign_key_fields (bool):
144                If True, returns full objects for foreign keys instead of just
145                their IDs. For example, 'created_by_id' will also return the
146                user object at 'created_by'. Defaults to False.
147            related_fields (bool):
148                If True, returns related objects (those that have a foreign key
149                pointing to this model). Results are typically returned as a
150                list of dictionaries in a field with a '_set' suffix.
151                Warning: Using this may consume significant backend resources.
152                Defaults to False.
153            use_disk_cache (bool):
154                If True, the GET request will use a local disk cache to reduce
155                backend load. Defaults to False.
156            use_app_cache (bool):
157                If True, the GET request will use the cache of the application.
158                Defaults to False.
159            disk_cache_expire (int, optional):
160                TTL in seconds for the cache. If None, uses the default
161                PumpwoodCache settings. Defaults to None.
162            base_filter_skip (list, optional):
163                List of base query filters to skip. Requires superuser
164                privileges. Defaults to None.
165
166        Returns:
167            dict: The object matching the provided primary key/identifier.
168
169        Raises:
170            PumpWoodObjectDoesNotExist:
171                If the PK is not found in the database.
172            PumpWoodException:
173                For other errors during retrieval or communication.
174        """
175        # Type checking and complex default values
176        is_allowed_types = isinstance(
177            pk, (numbers.Number, np.number, Decimal, str, dict))
178        if not is_allowed_types:
179            msg = (
180                "Retrieve pk must be a number, string or dict,"
181                " got type [{type}]")
182            raise PumpWoodException(
183                msg, payload={"type": type(pk).__name__})
184        base_filter_skip = (
185            [] if base_filter_skip is None else base_filter_skip)
186
187        # Convert to base64 dict unique queries
188        serialized_pk = None
189        if isinstance(pk, dict):
190            # Use the correct keyword argument 'primary_key_dict'
191            serialized_pk = CompositePkBase64Converter.dump_dict(
192                primary_key_dict=pk)
193        else:
194            serialized_pk = pk
195
196        # Fetch information from Pumpwood
197        url_str = self._build_retrieve_url(
198            model_class=model_class, pk=serialized_pk)
199        return self.request_get(
200            url=url_str, parameters={
201                "fields": fields, "default_fields": default_fields,
202                "foreign_key_fields": foreign_key_fields,
203                "related_fields": related_fields,
204                "base_filter_skip": base_filter_skip,
205                "use_cache": use_app_cache},
206            auth_header=auth_header, use_disk_cache=use_disk_cache,
207            disk_cache_expire=disk_cache_expire)
208
209    @staticmethod
210    def _build_retrieve_file_url(model_class: str, pk: int):
211        return "rest/%s/retrieve-file/%s/" % (model_class.lower(), pk)
212
213    def retrieve_file(self, model_class: str, pk: int, file_field: str,
214                      auth_header: dict = None, save_file: bool = True,
215                      save_path: str = "./", file_name: str = None,
216                      if_exists: str = "fail",
217                      base_filter_skip: list = None) -> any:
218        """Retrieve a file from PumpWood.
219
220        This function will retrieve file as a single request, depending on the
221        size of the files it would be preferred to use streaming end-point.
222
223        Args:
224            model_class:
225                Class of the model to retrieve file.
226            pk:
227                Pk of the object associeted file.
228            file_field:
229                Field of the file to be downloaded.
230            auth_header:
231                Dictionary containing the auth header.
232            save_file:
233                If data is to be saved as file or return get
234                response.
235            save_path:
236                Path of the directory to save file.
237            file_name:
238                Name of the file, if None it will have same name as
239                saved in PumpWood.
240            if_exists:
241                Values must be in {'fail', 'change_name', 'overwrite', 'skip'}.
242                Set what to do if there is a file with same name. Skip
243                will not download file if there is already with same
244                os.path.join(save_path, file_name), file_name must be set
245                for skip argument.
246            auth_header:
247                Auth header to substitute the microservice original
248                at the request (user impersonation).
249            base_filter_skip (list):
250                List of base query filter to be skiped, it is necessary to
251                be superuser to skip base query filters.
252
253        Returns:
254            May return the file name if save_file=True; If false will return
255            a dictonary with keys `filename` with original file name and
256            `content` with binary data of file content.
257
258        Raises:
259            PumpWoodForbidden:
260                'storage_object attribute not set for view, file operations
261                are disable'. This indicates that storage for this backend
262                was not configured, so it is not possible to make storage
263                operations,
264            PumpWoodForbidden:
265                'file_field must be set on self.file_fields dictionary'. This
266                indicates that the `file_field` parameter is not listed as
267                a file field on the backend.
268            PumpWoodObjectDoesNotExist:
269                'field [{}] not found or null at object'. This indicates that
270                the file field requested is not present on object fields.
271            PumpWoodObjectDoesNotExist:
272                'Object not found in storage [{}]'. This indicates that the
273                file associated with file_field is not avaiable at the
274                storage. This should not ocorrur, it might have a manual
275                update at the model_class table or manual removal/rename of
276                files on storage.
277        """
278        base_filter_skip = (
279            [] if base_filter_skip is None else base_filter_skip)
280
281        if if_exists not in ["fail", "change_name", "overwrite", "skip"]:
282            raise PumpWoodException(
283                "if_exists must be in ['fail', 'change_name', 'overwrite', "
284                "'skip']")
285
286        if file_name is not None and if_exists == 'skip':
287            file_path = os.path.join(save_path, file_name)
288            is_file_already = os.path.isfile(file_path)
289            if is_file_already:
290                print("skiping file already exists: ", file_path)
291                return file_path
292
293        url_str = self._build_retrieve_file_url(model_class=model_class, pk=pk)
294        file_response = self.request_get(
295            url=url_str, parameters={
296                "file-field": file_field,
297                "base_filter_skip": base_filter_skip},
298            auth_header=auth_header)
299        if not save_file:
300            return file_response
301
302        if not os.path.exists(save_path):
303            raise PumpWoodException(
304                "Path to save retrieved file [{}] does not exist".format(
305                    save_path))
306
307        file_name = secure_filename(file_name or file_response["filename"])
308        file_path = os.path.join(save_path, file_name)
309        is_file_already = os.path.isfile(file_path)
310        if is_file_already:
311            if if_exists == "change_name":
312                filename, file_extension = os.path.splitext(file_path)
313                too_many_tries = True
314                for i in range(10):
315                    new_path = "{filename}__{count}{extension}".format(
316                        filename=filename, count=i,
317                        extension=file_extension)
318                    if not os.path.isfile(new_path):
319                        file_path = new_path
320                        too_many_tries = False
321                        break
322                if too_many_tries:
323                    raise PumpWoodException(
324                        ("Too many tries to find a not used file name." +
325                         " file_path[{}]".format(file_path)))
326
327            elif if_exists == "fail":
328                raise PumpWoodException(
329                    ("if_exists set as 'fail' and there is a file with same" +
330                     "name. file_path [{}]").format(file_path))
331
332        with open(file_path, "wb") as file:
333            file.write(file_response["content"])
334        return file_path
335
336    @staticmethod
337    def _build_retrieve_file_straming_url(model_class: str, pk: int):
338        return "rest/%s/retrieve-file-streaming/%s/" % (
339            model_class.lower(), pk)
340
341    def retrieve_streaming_file(self, model_class: str, pk: int,
342                                file_field: str, file_name: str,
343                                auth_header: dict = None,
344                                save_path: str = "./",
345                                if_exists: str = "fail",
346                                base_filter_skip: list = None) -> str:
347        """Retrieve a file from PumpWood using streaming to retrieve content.
348
349        This funcion uses file streaming to retrieve file content, it should be
350        prefered when dealing with large (bigger than 10Mb) files transfer.
351        Using this end-point the file is not loaded on backend memory content
352        is transfered by chucks that are read at the storage and transfered
353        to user.
354
355        It will necessarily save the content as a file, there is not the
356        possibility of retrieving the content directly from request.
357
358        Args:
359            model_class:
360                Class of the model to retrieve file.
361            pk:
362                Pk of the object associeted file.
363            file_field:
364                Field of the file to be downloaded.
365            auth_header:
366                Dictionary containing the auth header.
367            save_path:
368                Path of the directory to save file.
369            file_name:
370                Name of the file, if None it will have same name as
371                saved in PumpWood.
372            if_exists:
373                Values must be in {'fail', 'change_name', 'overwrite'}.
374                Set what to do if there is a file with same name.
375            auth_header:
376                Auth header to substitute the microservice original
377                at the request (user impersonation).
378            base_filter_skip (list):
379                List of base query filter to be skiped, it is necessary to
380                be superuser to skip base query filters.
381
382        Returns:
383            Returns the file path that recived the file content.
384
385        Raises:
386            PumpWoodForbidden:
387                'storage_object attribute not set for view, file operations
388                are disable'. This indicates that storage for this backend
389                was not configured, so it is not possible to make storage
390                operations,
391            PumpWoodForbidden:
392                'file_field must be set on self.file_fields dictionary'. This
393                indicates that the `file_field` parameter is not listed as
394                a file field on the backend.
395            PumpWoodObjectDoesNotExist:
396                'field [{}] not found or null at object'. This indicates that
397                the file field requested is not present on object fields.
398            PumpWoodObjectDoesNotExist:
399                'Object not found in storage [{}]'. This indicates that the
400                file associated with file_field is not avaiable at the
401                storage. This should not ocorrur, it might have a manual
402                update at the model_class table or manual removal/rename of
403                files on storage.
404        """
405        base_filter_skip = (
406            [] if base_filter_skip is None else base_filter_skip)
407
408        request_header = self._check_auth_header(auth_header)
409
410        # begin Args check
411        if if_exists not in ["fail", "change_name", "overwrite"]:
412            raise PumpWoodException(
413                "if_exists must be in ['fail', 'change_name', 'overwrite']")
414
415        if not os.path.exists(save_path):
416            raise PumpWoodException(
417                "Path to save retrieved file [{}] does not exist".format(
418                    save_path))
419        # end Args check
420
421        file_path = os.path.join(save_path, file_name)
422        if os.path.isfile(file_path) and if_exists == "change_name":
423            filename, file_extension = os.path.splitext(file_path)
424            too_many_tries = False
425            for i in range(10):
426                new_path = "{filename}__{count}{extension}".format(
427                    filename=filename, count=i,
428                    extension=file_extension)
429                if not os.path.isfile(new_path):
430                    file_path = new_path
431                    too_many_tries = True
432                    break
433            if not too_many_tries:
434                raise PumpWoodException(
435                    ("Too many tries to find a not used file name." +
436                     " file_path[{}]".format(file_path)))
437
438        if os.path.isfile(file_path) and if_exists == "fail":
439            raise PumpWoodException(
440                ("if_exists set as 'fail' and there is a file with same" +
441                 "name. file_path [{}]").format(file_path))
442
443        url_str = self._build_retrieve_file_straming_url(
444            model_class=model_class, pk=pk)
445
446        get_url = self.server_url + url_str
447        with requests.get(
448                get_url, verify=self._verify_ssl, headers=request_header,
449                params={
450                    "file-field": file_field,
451                    "base_filter_skip": base_filter_skip},
452                timeout=self._default_timeout) as response:
453            self.error_handler(response)
454            with open(file_path, 'wb') as f:
455                for chunk in response.iter_content(chunk_size=8192):
456                    if chunk:
457                        f.write(chunk)
458        return file_path