pumpwood_communication.microservice_abc.simple

 21class ABCSimpleBatchMicroservice(ABC, PumpWoodMicroServiceBase):
 22    """Abstract class for batch end-points."""
 23
 24    @staticmethod
 25    def _build_aggregate_url(model_class: str):
 26        return "rest/%s/aggregate/" % (model_class.lower(),)
 27
 28    def aggregate(self, model_class: str, group_by: List[str] | str, agg: dict,
 29                  filter_dict: None | dict = None, exclude_dict: dict = None,
 30                  order_by: List[str] = None, auth_header: dict = None,
 31                  limit: int = None, as_dataframe: bool = False,
 32                  show_deleted: bool = False,
 33                  ) -> pd.DataFrame:
 34        """Save a list of objects with one request.
 35
 36        Avaiable aggregate functions:
 37            - **sum:** Sum.
 38            - **mean:** Mean.
 39            - **count:** Count.
 40            - **min:** Minimum.
 41            - **max:** Maximum.
 42            - **stddev_pop:** Population standard deviation.
 43            - **stddev_samp:** Sample standard deviation.
 44            - **var_pop:** Population variance.
 45            - **var_samp:** Sample variance.
 46            - **std:** Sample standard deviation.
 47            - **var:** Sample variance.
 48
 49        Args:
 50            model_class (str):
 51                Model class of the end-point that will be aggregated.
 52            group_by (List[str] | str):
 53                List of the fields that will be used on aggregation as
 54                group by. If a string is passed as argument, it will be
 55                considered the sigle group_by column.
 56            agg (dict):
 57                A dictionary with dictionary items as `field` and `function`
 58                specifing the field that will be aggregated using a function.
 59
 60                The dictinary keys will be used to return the results as
 61                columns.
 62            show_deleted (bool):
 63                If deleted data should be returned.
 64            filter_dict (dict):
 65                Filter that will be applied before the aggregation.
 66            exclude_dict (dict):
 67                Exclude clause that will be applied before the aggregation.
 68            order_by (list):
 69                Ordenation acording to grouping elements. It can be used
 70                fields created as keys of agg dictinary.
 71            auth_header (dict):
 72                Authentication header used to impersonation of user.
 73            limit (int):
 74                Limit number of returned row at aggregation query.
 75            as_dataframe (bool):
 76                If result should be returned as a dataframe. The columns
 77                will be set to match the group_by and agg arguments, this
 78                way empty dataframe with respect the columns.
 79
 80        Returns:
 81            Return a DataFrame with aggregation results.
 82
 83        Example:
 84            ```
 85            microservice.aggregate(
 86                model_class="ToLoadCalendar",
 87                group_by=["calendar_id"],
 88                agg={
 89                    "n": {"field": "id", "function": "count"},
 90                    "mean": {"field": "value", "function": "mean"
 91                }})
 92            ```
 93        """
 94        filter_dict = {} if filter_dict is None else filter_dict
 95        exclude_dict = {} if exclude_dict is None else exclude_dict
 96        order_by = [] if order_by is None else order_by
 97
 98        # If group_by is a string, convert to a list with this string
 99        group_by = [group_by] if isinstance(group_by, str) else group_by
100        if not isinstance(group_by, (list, tuple, set)):
101            error_msg = "Argument `group_by` must be list, tuple, set or str."
102            raise TypeError(error_msg)
103        if not isinstance(agg, (dict)):
104            error_msg = "Argument `agg` must be dict."
105            raise TypeError(error_msg)
106
107        url_str = self._build_aggregate_url(model_class=model_class)
108        data = {
109            'agg': agg, 'group_by': group_by, 'filter_dict': filter_dict,
110            'exclude_dict': exclude_dict, 'order_by': order_by,
111            'limit': limit, 'show_deleted': show_deleted}
112        return_data = self.request_post(
113            url=url_str, data=data, auth_header=auth_header)
114        if not as_dataframe:
115            return return_data
116        else:
117            # Return the results as a dataframe using the group_by columns
118            # and the columns created at aggregation to ensure that
119            # even empty results would have the correct columns
120            return_columns = group_by + list(agg.keys())
121            return pd.DataFrame(return_data, columns=return_columns)
122
123    @staticmethod
124    def _build_pivot_url(model_class):
125        return "rest/%s/pivot/" % (model_class.lower(), )
126
127    def pivot(self, model_class: str, columns: None | List[str] = None,
128              format: str = 'list', filter_dict: dict = None,
129              exclude_dict: dict = None, order_by: List[str] = None,
130              variables: List[str] = None, fields: List[str] = None,
131              show_deleted: bool = False,
132              add_pk_column: bool = False, auth_header: dict = None,
133              as_dataframe: bool = False
134              ) -> Union[List[dict], Dict[str, list], pd.DataFrame]:
135        """Pivot object data acording to columns specified.
136
137        Pivoting per-se is not usually used, beeing the name of the function
138        a legacy. Normality data transformation is done at the client level.
139
140        Args:
141            model_class (str):
142                Model class to check search parameters.
143            columns (List[str]):
144                List of fields to be used as columns when pivoting the data.
145            format (str):
146                Format to be used to convert pandas.DataFrame to
147                dictionary, must be in ['dict','list','series',
148                'split', 'records','index'].
149            filter_dict (dict):
150                Same as list function.
151            exclude_dict (dict):
152                Same as list function.
153            order_by (List[str]):
154                 Same as list function.
155            fields (List[str]):
156                List of the fields to be returned, if None, the default
157                variables will be returned. Same as fields on list functions.
158            variables (List[str]):
159                List of the fields to be returned, if None, the default
160                variables will be returned. Same as fields on list functions.
161                **DEPRECTED** use fields.
162            show_deleted (bool):
163                Fields with deleted column will have objects with deleted=True
164                omited from results. show_deleted=True will return this
165                information.
166            add_pk_column (bool):
167                If add pk values of the objects at pivot results. Adding
168                pk key on pivot end-points won't be possible to pivot since
169                pk is unique for each entry.
170            auth_header (dict):
171                Auth header to substitute the microservice original
172                at the request (user impersonation).
173            as_dataframe (bool):
174                If results should be returned as a dataframe.
175
176        Returns:
177            Return a list or a dictinary depending on the format set on
178            format parameter.
179
180        Raises:
181            PumpWoodException:
182                'Columns must be a list of elements.'. Indicates that the list
183                argument was not a list.
184            PumpWoodException:
185                'Column chosen as pivot is not at model variables'. Indicates
186                that columns that were set to pivot are not present on model
187                variables.
188            PumpWoodException:
189                "Format must be in ['dict','list','series','split',
190                'records','index']". Indicates that format set as paramenter
191                is not implemented.
192            PumpWoodException:
193                "Can not add pk column and pivot information". If
194                add_pk_column is True (results will have the pk column), it is
195                not possible to pivot the information (pk is an unique value
196                for each object, there is no reason to pivot it).
197            PumpWoodException:
198                "'value' column not at melted data, it is not possible
199                to pivot dataframe.". Indicates that data does not have a value
200                column, it must have it to populate pivoted table.
201        """
202        # Deprect variables argument
203        if variables is not None:
204            msg = (
205                "Deprecation Warning: pivot variable is deprected, "
206                "you should use fields instead")
207            logger.warning(msg)
208        fields = fields if fields is not None else variables
209
210        filter_dict = {} if filter_dict is None else filter_dict
211        exclude_dict = {} if exclude_dict is None else exclude_dict
212        order_by = [] if order_by is None else order_by
213        columns = [] if columns is None else columns
214
215        url_str = self._build_pivot_url(model_class)
216        post_data = {
217            'columns': columns, 'format': format,
218            'filter_dict': filter_dict, 'exclude_dict': exclude_dict,
219            'order_by': order_by, "variables": variables,
220            "show_deleted": show_deleted, "add_pk_column": add_pk_column}
221        pivot_results = self.request_post(
222            url=url_str, data=post_data, auth_header=auth_header)
223
224        if not add_pk_column:
225            if as_dataframe:
226                # If passed, set the fields as columns to ensure empty
227                # dataframes to have the correct fields
228                return pd.DataFrame(pivot_results, columns=fields)
229            else:
230                return pivot_results
231        else:
232            # If passed, set the fields as columns to ensure empty
233            # dataframes to have the correct fields
234            pd_pivot_results = pd.DataFrame(pivot_results, columns=fields)
235            if len(pd_pivot_results) != 0:
236                fill_options = self.fill_options(
237                    model_class=model_class, auth_header=auth_header)
238                primary_keys_data = fill_options["pk"]
239                # Allow legacy pk definition and actual that uses extra_info
240                primary_keys = primary_keys_data\
241                    .get('extra_info', {})\
242                    .get('columns', primary_keys_data['column'])
243                pd_pivot_results["pk"] = pd_pivot_results[primary_keys].apply(
244                    CompositePkBase64Converter.dump,
245                    primary_keys=primary_keys, axis=1)
246            if as_dataframe:
247                return pd_pivot_results
248            else:
249                return pd_pivot_results.to_dict(format)
250
251    def _flat_list_by_chunks_helper(self, args):
252        try:
253            # Unpacking arguments
254            model_class = args["model_class"]
255            filter_dict = args["filter_dict"]
256            exclude_dict = args["exclude_dict"]
257            fields = args["fields"]
258            show_deleted = args["show_deleted"]
259            auth_header = args["auth_header"]
260            chunk_size = args["chunk_size"]
261
262            temp_filter_dict = copy.deepcopy(filter_dict)
263            url_str = self._build_pivot_url(model_class)
264            max_pk = 0
265
266            # Fetch data until an empty result is returned
267            list_dataframes = []
268            while True:
269                sys.stdout.write(".")
270                sys.stdout.flush()
271                temp_filter_dict["id__gt"] = max_pk
272                post_data = {
273                    'format': 'list',
274                    'filter_dict': temp_filter_dict,
275                    'exclude_dict': exclude_dict,
276                    'order_by': ["id"], "variables": fields,
277                    "show_deleted": show_deleted,
278                    "limit": chunk_size,
279                    "add_pk_column": True}
280                temp_dateframe = pd.DataFrame(self.request_post(
281                    url=url_str, data=post_data, auth_header=auth_header))
282
283                # Break if results are less than chunk size, so no more results
284                # are avaiable
285                if len(temp_dateframe) < chunk_size:
286                    list_dataframes.append(temp_dateframe)
287                    break
288
289                max_pk = int(temp_dateframe["id"].max())
290                list_dataframes.append(temp_dateframe)
291
292            if len(list_dataframes) == 0:
293                return pd.DataFrame()
294            else:
295                return pd.concat(list_dataframes)
296        except Exception as e:
297            raise Exception("Exception at flat_list_by_chunks:", str(e))
298
299    def flat_list_by_chunks(self, model_class: str, filter_dict: dict = None,
300                            exclude_dict: dict = None,
301                            fields: List[str] = None,
302                            show_deleted: bool = False,
303                            auth_header: dict = None,
304                            chunk_size: int = 1000000,
305                            n_parallel: int = None,
306                            create_composite_pk: bool = False,
307                            start_date: str = None,
308                            end_date: str = None,
309                            time_column: str = 'time') -> pd.DataFrame:
310        """Incrementally fetch data from pivot end-point.
311
312        Fetch data from pivot end-point paginating by id of chunk_size length.
313
314        If table is partitioned it will split the query acording to partition
315        to facilitate query at the database.
316
317        If start_date and end_date are set, also breaks the query by month
318        retrieving each month data in parallel.
319
320        Args:
321            model_class (str):
322                Model class to be pivoted.
323            filter_dict (dict):
324                Dictionary to to be used in objects.filter argument
325                (Same as list end-point).
326            exclude_dict (dict):
327                Dictionary to to be used in objects.exclude argument
328                (Same as list end-point).
329            fields (List[str] | None):
330                List of the variables to be returned,
331                if None, the default variables will be returned.
332                If fields is set, dataframe will return that columns
333                even if data is empty.
334            start_date (datetime | str):
335                Set a begin date for the query. If begin and end date are
336                set, query will be splited with chucks by month that will be
337                requested in parallel.
338            end_date (datetime | str):
339                Set a end date for the query. If begin and end date are
340                set, query will be splited with chucks by month that will be
341                requested in parallel.
342            show_deleted (bool):
343                If deleted data should be returned.
344            auth_header (dict):
345                Auth header to substitute the microservice original
346                at the request (user impersonation).
347            chunk_size (int):
348                Limit of data to fetch per call.
349            n_parallel (int):
350                Number of parallel process to perform.
351            create_composite_pk (bool):
352                If true and table has a composite pk, it will create pk
353                value based on the hash on the json serialized dictionary
354                of the components of the primary key.
355            time_column (str):
356                Column that will be considered on the partitioning of data
357                fetch.
358
359        Returns:
360            Returns a dataframe with all information fetched.
361
362        Raises:
363            No particular raise.
364        """
365        # Set empty dictionary for dictionary arguments
366        filter_dict = {} if filter_dict is None else filter_dict
367        exclude_dict = {} if exclude_dict is None else exclude_dict
368        original_fields = None if fields is None else fields
369
370        if n_parallel is None:
371            n_parallel = PUMPWOOD_COMUNICATION__N_PARALLEL
372
373        fill_options = self.fill_options(
374            model_class=model_class, auth_header=auth_header)
375
376        # Retrieve PK infomration about primary_key and partitions
377        primary_keys = fill_options["pk"]["extra_info"]['columns']
378        # partition = fill_options["pk"]["extra_info"]['partition']
379
380        # Add all primary keys fields to query if create_composite_pk is
381        # set true
382        if create_composite_pk and fields is not None:
383            fields = list(set(fields) | set(primary_keys))
384
385        # Create a list of month and include start and end dates if not at
386        # the beginning of a month
387        month_sequence = None
388        if (start_date is not None) and (end_date is not None):
389            month_sequence = AuxFlatListByChunks.build_month_partitions(
390                start_date=start_date,
391                end_date=end_date)
392        elif (start_date is not None) or (end_date is not None):
393            # To create the partitions is necessary to have both start and end
394            # date.
395            msg = (
396                "To break query in chunks using start_date and end_date "
397                "both must be set.\n- start_date: {start_date}\n"
398                "- end_date: {end_date}")
399            raise PumpWoodException(
400                message=msg, payload={
401                    "start_date": start_date,
402                    "end_date": end_date})
403
404        pool_arguments = AuxFlatListByChunks.build_query_partitions(
405            model_class=model_class, time_partitions=month_sequence,
406            filter_dict=filter_dict, exclude_dict=exclude_dict, fields=fields,
407            time_column=time_column, show_deleted=show_deleted,
408            auth_header=auth_header, chunk_size=chunk_size)
409
410        # Perform parallel calls to backend each chucked by chunk_size
411        print("## Starting parallel flat list: %s" % len(pool_arguments))
412        resp_df = None
413        try:
414            with Pool(n_parallel) as p:
415                results = p.map(
416                    self._flat_list_by_chunks_helper,
417                    pool_arguments)
418            if len(results) != 0:
419                resp_df = pd.concat(results)\
420                    .reset_index(drop=True)
421            else:
422                resp_df = pd.DataFrame(columns=fields)
423        except Exception as e:
424            PumpWoodException(message=str(e))
425        print("\n## Finished parallel flat list: %s" % len(pool_arguments))
426
427        # Add the primary key as a column to the dataframe
428        resp_df = AuxFlatListByChunks.add_pk_column(
429            create_composite_pk=create_composite_pk,
430            primary_key_list=primary_keys,
431            data=resp_df)
432
433        # Limit the return fields
434        if original_fields is not None:
435            if create_composite_pk:
436                return pd.DataFrame(
437                    resp_df, columns=['pk'] + original_fields)
438            else:
439                return pd.DataFrame(
440                    resp_df, columns=original_fields)
441        else:
442            return resp_df
443
444    def get_pks_from_unique_field(self, model_class: str, field: str,
445                                  values: List[Any]) -> pd.DataFrame:
446        """Get pk using unique fields values.
447
448        Use unique field values to retrieve pk of the objects. This end-point
449        is usefull for retrieving pks of the objects associated with unique
450        fields such as `description` (unique on most model of pumpwood).
451
452        ```python
453        # Using description to fetch pks from objects
454        data: pd.DataFrame = [data with unique description but without pk]
455        data['attribute_id'] = microservice.get_pks_from_unique_field(
456            model_class="DescriptionAttribute",
457            field="description", values=data['attribute'])['pk']
458
459        # Using a dimension key to fetch pk of the objects, dimension
460        # key must be unique
461        data['georea_id'] = microservice.get_pks_from_unique_field(
462            model_class="DescriptionGeoarea", field="dimension->city",
463            values=data['city'])['pk']
464        ```
465
466        Args:
467            model_class:
468                Model class of the objects.
469            field:
470                Unique field to fetch pk. It is possible to use dimension keys
471                as unique field, for that use `dimension->[key]` notation.
472            values:
473                List of the unique fields used to fetch primary keys.
474
475        Return:
476            Return a dataframe in same order as values with columns:
477            - **pk**: Correspondent primary key of the unique value.
478            - **[field]**: Column with same name of field argument,
479                correspondent to pk.
480
481        Raises:
482            PumpWoodQueryException:
483                Raises if field is not found on the model and it is note
484                associated with a dimension tag.
485            PumpWoodQueryException:
486                Raises if `field` does not have a unique restriction on
487                database. Dimension keys does not check for uniqueness on
488                database, be carefull not to duplicate the lines.
489        """
490        is_dimension_tag = 'dimensions->' in field
491        if not is_dimension_tag:
492            fill_options = self.fill_options(model_class=model_class)
493            field_details = fill_options.get(field)
494            if field_details is None:
495                msg = (
496                    "Field is not a dimension tag and not found on model "
497                    "fields. Field [{field}]")
498                raise PumpWoodQueryException(
499                    message=msg, payload={"field": field})
500
501            is_unique_field = field_details.get("unique", False)
502            if not is_unique_field:
503                msg = "Field [{field}] to get pk from is not unique"
504                raise PumpWoodQueryException(
505                    message=msg, payload={"field": field})
506
507        filter_dict = {field + "__in": list(set(values))}
508        pk_map = None
509        if not is_dimension_tag:
510            list_results = pd.DataFrame(self.list_without_pag(
511                model_class=model_class, filter_dict=filter_dict,
512                fields=["pk", field]), columns=["pk", field])
513            pk_map = list_results.set_index(field)["pk"]
514
515        # If is dimension tag, fetch dimension and unpack it
516        else:
517            dimension_tag = field.split("->")[1]
518            list_results = pd.DataFrame(self.list_without_pag(
519                model_class=model_class, filter_dict=filter_dict,
520                fields=["pk", "dimensions"]))
521            pk_map = {}
522            if len(list_results) != 0:
523                pk_map = list_results\
524                    .pipe(unpack_dict_columns, columns=["dimensions"])\
525                    .set_index(dimension_tag)["pk"]
526
527        values_series = pd.Series(values)
528        return pd.DataFrame({
529            "pk": values_series.map(pk_map).to_numpy(),
530            field: values_series
531        })

 16class ABCSimpleRetriveMicroservice(ABC, PumpWoodMicroServiceBase):
 17    """Abstract class for parallel calls at Pumpwood end-points."""
 18
 19    @staticmethod
 20    def _build_list_one_url(model_class, pk):
 21        return "rest/%s/retrieve/%s/" % (model_class.lower(), pk)
 22
 23    def list_one(self, model_class: str, pk: int, fields: list = None,
 24                 default_fields: bool = True, foreign_key_fields: bool = False,
 25                 related_fields: bool = False, auth_header: dict = None,
 26                 use_disk_cache: bool = False, use_app_cache: bool = False,
 27                 disk_cache_expire: int = None,
 28                 base_filter_skip: list = None) -> dict:
 29        """Retrieve an object using list serializer (simple).
 30
 31        **# DEPRECTED #** It is the same as retrieve using
 32        `default_fields: bool = True`, if possible migrate to retrieve
 33        function.
 34
 35        Args:
 36            model_class:
 37                Model class of the end-point
 38            pk:
 39                Object pk
 40            auth_header:
 41                Auth header to substitute the microservice original
 42                at the request (user impersonation).
 43            fields:
 44                Set the fields to be returned by the list end-point.
 45            default_fields:
 46                Boolean, if true and fields arguments None will return the
 47                default fields set for list by the backend.
 48            foreign_key_fields:
 49                Return forenging key objects. It will return the fk
 50                corresponding object. Ex: `created_by_id` reference to
 51                a user `model_class` the correspondent to User will be
 52                returned at `created_by`.
 53            related_fields:
 54                Return related fields objects. Related field objects are
 55                objects that have a forenging key associated with this
 56                model_class, results will be returned as a list of
 57                dictionaries usually in a field with `_set` at end.
 58                Returning related_fields consume backend resorces, use
 59                carefully.
 60            use_disk_cache (bool):
 61                If set true, get request will use local cache to reduce
 62                the requests to the backend.
 63            use_app_cache (bool):
 64                If True, the GET request will use the cache of the application.
 65                Defaults to False.
 66            disk_cache_expire (int):
 67                Time in seconds to expire the cache, it None it will
 68                use de default set be PumpwoodCache.
 69            base_filter_skip (list[str]):
 70                List of base query filter to be skiped, it is necessary to
 71                be superuser to skip base query filters.
 72
 73        Returns:
 74            Return object with the correspondent pk.
 75
 76        Raises:
 77            PumpWoodObjectDoesNotExist:
 78                If pk not found on database.
 79        """
 80        base_filter_skip = self._resolve_base_filter_skip(
 81            base_filter_skip)
 82        url_str = self._build_list_one_url(model_class, pk)
 83        return self.request_get(
 84            url=url_str, parameters={
 85                "fields": fields, "default_fields": default_fields,
 86                "foreign_key_fields": foreign_key_fields,
 87                "related_fields": related_fields,
 88                "base_filter_skip": base_filter_skip,
 89                "use_cache": use_app_cache},
 90            auth_header=auth_header, use_disk_cache=use_disk_cache,
 91            disk_cache_expire=disk_cache_expire)
 92
 93    @staticmethod
 94    def _build_retrieve_url(model_class: str, pk: int):
 95        return "rest/%s/retrieve/%s/" % (model_class.lower(), pk)
 96
 97    def retrieve(self, model_class: str, pk: int | str | dict,
 98                 default_fields: bool = False,
 99                 foreign_key_fields: bool = False,
100                 related_fields: bool = False,
101                 fields: list = None,
102                 auth_header: dict = None,
103                 use_disk_cache: bool = False,
104                 use_app_cache: bool = False,
105                 disk_cache_expire: int = None,
106                 base_filter_skip: list = None) -> dict:
107        """Retrieve an object from PumpWood.
108
109        Function to get an object serialized by the retrieve endpoint (more
110        detailed data). It will fetch information for a single object
111        based on the primary key, which may be a simple ID, a composite key
112        passed as a dictionary, or a base64 URL-safe string.
113
114        It is also possible to retrieve single objects using unique fields,
115        such as codes or multiple column uniqueness constraints. This can
116        be done by passing the argument as a base64 string or a dictionary
117        containing the filtering clauses.
118
119        Example:
120            ```python
121            microservice.retrieve(
122                model_class="ModelClassWithUniqueCode",
123                pk={'code': 'unique code for object'})
124
125            microservice.retrieve(
126                model_class="ModelClassWithCompositeUniqueConstraint",
127                pk={'time': '2026-01-01', 'attribute_id': 1})
128            ```
129
130        Args:
131            model_class (str):
132                Model class of the endpoint.
133            pk (Union[int, str, dict]):
134                The primary key or unique identifier for the object.
135            auth_header (dict, optional):
136                Authentication header to substitute the microservice's original
137                credentials (used for user impersonation). Defaults to None.
138            fields (list, optional):
139                Set of fields to be returned by the endpoint.
140            default_fields (bool):
141                If True and 'fields' is None, will return the default fields
142                defined by the backend. Defaults to False.
143            foreign_key_fields (bool):
144                If True, returns full objects for foreign keys instead of just
145                their IDs. For example, 'created_by_id' will also return the
146                user object at 'created_by'. Defaults to False.
147            related_fields (bool):
148                If True, returns related objects (those that have a foreign key
149                pointing to this model). Results are typically returned as a
150                list of dictionaries in a field with a '_set' suffix.
151                Warning: Using this may consume significant backend resources.
152                Defaults to False.
153            use_disk_cache (bool):
154                If True, the GET request will use a local disk cache to reduce
155                backend load. Defaults to False.
156            use_app_cache (bool):
157                If True, the GET request will use the cache of the application.
158                Defaults to False.
159            disk_cache_expire (int, optional):
160                TTL in seconds for the cache. If None, uses the default
161                PumpwoodCache settings. Defaults to None.
162            base_filter_skip (list, optional):
163                List of base query filters to skip. Requires superuser
164                privileges. Defaults to None.
165
166        Returns:
167            dict: The object matching the provided primary key/identifier.
168
169        Raises:
170            PumpWoodObjectDoesNotExist:
171                If the PK is not found in the database.
172            PumpWoodException:
173                For other errors during retrieval or communication.
174        """
175        # Type checking and complex default values
176        is_allowed_types = isinstance(
177            pk, (numbers.Number, np.number, Decimal, str, dict))
178        if not is_allowed_types:
179            msg = (
180                "Retrieve pk must be a number, string or dict,"
181                " got type [{type}]")
182            raise PumpWoodException(
183                msg, payload={"type": type(pk).__name__})
184        
185        base_filter_skip = self._resolve_base_filter_skip(
186            base_filter_skip)
187
188        # Convert to base64 dict unique queries
189        serialized_pk = None
190        if isinstance(pk, dict):
191            # Use the correct keyword argument 'primary_key_dict'
192            serialized_pk = CompositePkBase64Converter.dump_dict(
193                primary_key_dict=pk)
194        else:
195            serialized_pk = pk
196
197        # Fetch information from Pumpwood
198        url_str = self._build_retrieve_url(
199            model_class=model_class, pk=serialized_pk)
200        return self.request_get(
201            url=url_str, parameters={
202                "fields": fields, "default_fields": default_fields,
203                "foreign_key_fields": foreign_key_fields,
204                "related_fields": related_fields,
205                "base_filter_skip": base_filter_skip,
206                "use_cache": use_app_cache},
207            auth_header=auth_header, use_disk_cache=use_disk_cache,
208            disk_cache_expire=disk_cache_expire)
209
210    @staticmethod
211    def _build_retrieve_file_url(model_class: str, pk: int):
212        return "rest/%s/retrieve-file/%s/" % (model_class.lower(), pk)
213
214    def retrieve_file(self, model_class: str, pk: int, file_field: str,
215                      auth_header: dict = None, save_file: bool = True,
216                      save_path: str = "./", file_name: str = None,
217                      if_exists: str = "fail",
218                      base_filter_skip: list = None) -> any:
219        """Retrieve a file from PumpWood.
220
221        This function will retrieve file as a single request, depending on the
222        size of the files it would be preferred to use streaming end-point.
223
224        Args:
225            model_class:
226                Class of the model to retrieve file.
227            pk:
228                Pk of the object associeted file.
229            file_field:
230                Field of the file to be downloaded.
231            auth_header:
232                Dictionary containing the auth header.
233            save_file:
234                If data is to be saved as file or return get
235                response.
236            save_path:
237                Path of the directory to save file.
238            file_name:
239                Name of the file, if None it will have same name as
240                saved in PumpWood.
241            if_exists:
242                Values must be in {'fail', 'change_name', 'overwrite', 'skip'}.
243                Set what to do if there is a file with same name. Skip
244                will not download file if there is already with same
245                os.path.join(save_path, file_name), file_name must be set
246                for skip argument.
247            auth_header:
248                Auth header to substitute the microservice original
249                at the request (user impersonation).
250            base_filter_skip (list):
251                List of base query filter to be skiped, it is necessary to
252                be superuser to skip base query filters.
253
254        Returns:
255            May return the file name if save_file=True; If false will return
256            a dictonary with keys `filename` with original file name and
257            `content` with binary data of file content.
258
259        Raises:
260            PumpWoodForbidden:
261                'storage_object attribute not set for view, file operations
262                are disable'. This indicates that storage for this backend
263                was not configured, so it is not possible to make storage
264                operations,
265            PumpWoodForbidden:
266                'file_field must be set on self.file_fields dictionary'. This
267                indicates that the `file_field` parameter is not listed as
268                a file field on the backend.
269            PumpWoodObjectDoesNotExist:
270                'field [{}] not found or null at object'. This indicates that
271                the file field requested is not present on object fields.
272            PumpWoodObjectDoesNotExist:
273                'Object not found in storage [{}]'. This indicates that the
274                file associated with file_field is not avaiable at the
275                storage. This should not ocorrur, it might have a manual
276                update at the model_class table or manual removal/rename of
277                files on storage.
278        """
279        base_filter_skip = self._resolve_base_filter_skip(
280            base_filter_skip)
281
282        if if_exists not in ["fail", "change_name", "overwrite", "skip"]:
283            raise PumpWoodException(
284                "if_exists must be in ['fail', 'change_name', 'overwrite', "
285                "'skip']")
286
287        if file_name is not None and if_exists == 'skip':
288            file_path = os.path.join(save_path, file_name)
289            is_file_already = os.path.isfile(file_path)
290            if is_file_already:
291                print("skiping file already exists: ", file_path)
292                return file_path
293
294        url_str = self._build_retrieve_file_url(model_class=model_class, pk=pk)
295        file_response = self.request_get(
296            url=url_str, parameters={
297                "file-field": file_field,
298                "base_filter_skip": base_filter_skip},
299            auth_header=auth_header)
300        if not save_file:
301            return file_response
302
303        if not os.path.exists(save_path):
304            raise PumpWoodException(
305                "Path to save retrieved file [{}] does not exist".format(
306                    save_path))
307
308        file_name = secure_filename(file_name or file_response["filename"])
309        file_path = os.path.join(save_path, file_name)
310        is_file_already = os.path.isfile(file_path)
311        if is_file_already:
312            if if_exists == "change_name":
313                filename, file_extension = os.path.splitext(file_path)
314                too_many_tries = True
315                for i in range(10):
316                    new_path = "{filename}__{count}{extension}".format(
317                        filename=filename, count=i,
318                        extension=file_extension)
319                    if not os.path.isfile(new_path):
320                        file_path = new_path
321                        too_many_tries = False
322                        break
323                if too_many_tries:
324                    raise PumpWoodException(
325                        ("Too many tries to find a not used file name." +
326                         " file_path[{}]".format(file_path)))
327
328            elif if_exists == "fail":
329                raise PumpWoodException(
330                    ("if_exists set as 'fail' and there is a file with same" +
331                     "name. file_path [{}]").format(file_path))
332
333        with open(file_path, "wb") as file:
334            file.write(file_response["content"])
335        return file_path
336
337    @staticmethod
338    def _build_retrieve_file_straming_url(model_class: str, pk: int):
339        return "rest/%s/retrieve-file-streaming/%s/" % (
340            model_class.lower(), pk)
341
342    def retrieve_streaming_file(self, model_class: str, pk: int,
343                                file_field: str, file_name: str,
344                                auth_header: dict = None,
345                                save_path: str = "./",
346                                if_exists: str = "fail",
347                                base_filter_skip: list = None) -> str:
348        """Retrieve a file from PumpWood using streaming to retrieve content.
349
350        This funcion uses file streaming to retrieve file content, it should be
351        prefered when dealing with large (bigger than 10Mb) files transfer.
352        Using this end-point the file is not loaded on backend memory content
353        is transfered by chucks that are read at the storage and transfered
354        to user.
355
356        It will necessarily save the content as a file, there is not the
357        possibility of retrieving the content directly from request.
358
359        Args:
360            model_class:
361                Class of the model to retrieve file.
362            pk:
363                Pk of the object associeted file.
364            file_field:
365                Field of the file to be downloaded.
366            auth_header:
367                Dictionary containing the auth header.
368            save_path:
369                Path of the directory to save file.
370            file_name:
371                Name of the file, if None it will have same name as
372                saved in PumpWood.
373            if_exists:
374                Values must be in {'fail', 'change_name', 'overwrite'}.
375                Set what to do if there is a file with same name.
376            auth_header:
377                Auth header to substitute the microservice original
378                at the request (user impersonation).
379            base_filter_skip (list):
380                List of base query filter to be skiped, it is necessary to
381                be superuser to skip base query filters.
382
383        Returns:
384            Returns the file path that recived the file content.
385
386        Raises:
387            PumpWoodForbidden:
388                'storage_object attribute not set for view, file operations
389                are disable'. This indicates that storage for this backend
390                was not configured, so it is not possible to make storage
391                operations,
392            PumpWoodForbidden:
393                'file_field must be set on self.file_fields dictionary'. This
394                indicates that the `file_field` parameter is not listed as
395                a file field on the backend.
396            PumpWoodObjectDoesNotExist:
397                'field [{}] not found or null at object'. This indicates that
398                the file field requested is not present on object fields.
399            PumpWoodObjectDoesNotExist:
400                'Object not found in storage [{}]'. This indicates that the
401                file associated with file_field is not avaiable at the
402                storage. This should not ocorrur, it might have a manual
403                update at the model_class table or manual removal/rename of
404                files on storage.
405        """
406        base_filter_skip = self._resolve_base_filter_skip(
407            base_filter_skip)
408        request_header = self._check_auth_header(auth_header)
409
410        if if_exists not in ["fail", "change_name", "overwrite"]:
411            raise PumpWoodException(
412                "if_exists must be in ['fail', 'change_name', 'overwrite']")
413        if not os.path.exists(save_path):
414            raise PumpWoodException(
415                "Path to save retrieved file [{}] does not exist".format(
416                    save_path))
417
418        file_path = os.path.join(save_path, file_name)
419        if os.path.isfile(file_path) and if_exists == "change_name":
420            filename, file_extension = os.path.splitext(file_path)
421            too_many_tries = False
422            for i in range(10):
423                new_path = "{filename}__{count}{extension}".format(
424                    filename=filename, count=i,
425                    extension=file_extension)
426                if not os.path.isfile(new_path):
427                    file_path = new_path
428                    too_many_tries = True
429                    break
430            if not too_many_tries:
431                raise PumpWoodException(
432                    ("Too many tries to find a not used file name." +
433                     " file_path[{}]".format(file_path)))
434
435        if os.path.isfile(file_path) and if_exists == "fail":
436            raise PumpWoodException(
437                ("if_exists set as 'fail' and there is a file with same" +
438                 "name. file_path [{}]").format(file_path))
439
440        url_str = self._build_retrieve_file_straming_url(
441            model_class=model_class, pk=pk)
442
443        get_url = self.server_url + url_str
444        get_params = {
445            "file-field": file_field,
446            "base_filter_skip": base_filter_skip}
447        dumped_parameters = self._dump_query_parameters(parameters=get_params)
448        with requests.get(
449                get_url, verify=self._verify_ssl, headers=request_header,
450                params=dumped_parameters,
451                timeout=self._default_timeout) as response:
452            self.error_handler(response)
453            with open(file_path, 'wb') as f:
454                for chunk in response.iter_content(chunk_size=8192):
455                    if chunk:
456                        f.write(chunk)
457        return file_path