import abc import codecs import json import re from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from rest_framework import status from rest_framework.exceptions import ParseError, APIException from rest_framework.parsers import BaseParser from common.serializers.fields import ObjectRelatedField, LabeledChoiceField from common.utils import get_logger logger = get_logger(__file__) class FileContentOverflowedError(APIException): status_code = status.HTTP_400_BAD_REQUEST default_code = 'file_content_overflowed' default_detail = _('The file content overflowed (The maximum length `{}` bytes)') class BaseFileParser(BaseParser): FILE_CONTENT_MAX_LENGTH = 1024 * 1024 * 10 serializer_cls = None serializer_fields = None obj_pattern = re.compile(r'^(.+)\(([a-z0-9-]+)\)$') def check_content_length(self, meta): content_length = int(meta.get('CONTENT_LENGTH', meta.get('HTTP_CONTENT_LENGTH', 0))) if content_length > self.FILE_CONTENT_MAX_LENGTH: msg = FileContentOverflowedError.default_detail.format(self.FILE_CONTENT_MAX_LENGTH) logger.error(msg) raise FileContentOverflowedError(msg) @staticmethod def get_stream_data(stream): stream_data = stream.read() stream_data = stream_data.strip(codecs.BOM_UTF8) return stream_data @abc.abstractmethod def generate_rows(self, stream_data): raise NotImplementedError def get_column_titles(self, rows): return next(rows) def convert_to_field_names(self, column_titles): fields_map = {} fields = self.serializer_fields for k, v in fields.items(): # 资产平台的 id 是只读的, 导入更新资产平台会失败 if v.read_only and k not in ['id', 'pk']: continue fields_map.update({ v.label: k, k: k }) lowercase_fields_map = {k.lower(): v for k, v in fields_map.items()} field_names = [ lowercase_fields_map.get(column_title.strip('*').lower(), '') for column_title in column_titles ] return field_names @staticmethod def _replace_chinese_quote(s): if not isinstance(s, str): return s trans_table = str.maketrans({ '“': '"', '”': '"', '‘': '"', '’': '"', '\'': '"' }) return s.translate(trans_table) @classmethod def load_row(cls, row): """ 构建json数据前的行处理 """ new_row = [] for col in row: # 转换中文引号 col = cls._replace_chinese_quote(col) # 列表/字典转换 if isinstance(col, str) and ( (col.startswith('[') and col.endswith(']')) or (col.startswith("{") and col.endswith("}")) ): try: col = json.loads(col) except json.JSONDecodeError as e: logger.error('Json load error: ', e) logger.error('col: ', col) new_row.append(col) return new_row def id_name_to_obj(self, v): if not v or not isinstance(v, str): return v matched = self.obj_pattern.match(v) if not matched: return v obj_name, obj_id = matched.groups() if len(obj_id) < 36: obj_id = int(obj_id) return {'pk': obj_id, 'name': obj_name} def parse_value(self, field, value): if value == '-' and field and field.allow_null: return None elif hasattr(field, 'to_file_internal_value'): value = field.to_file_internal_value(value) elif isinstance(field, serializers.BooleanField): value = value.lower() in ['true', '1', 'yes'] elif isinstance(field, ObjectRelatedField): if field.many: value = [self.id_name_to_obj(v) for v in value] else: value = self.id_name_to_obj(value) elif isinstance(field, LabeledChoiceField): value = self.id_name_to_obj(value) if isinstance(value, dict) and value.get('pk'): value = value.get('pk') elif isinstance(field, serializers.ListSerializer): value = [self.parse_value(field.child, v) for v in value] elif isinstance(field, serializers.Serializer): value = self.id_name_to_obj(value) elif isinstance(field, serializers.ManyRelatedField): value = [self.parse_value(field.child_relation, v) for v in value] elif isinstance(field, serializers.ListField): value = [self.parse_value(field.child, v) for v in value] return value def process_row_data(self, row_data): """ 构建json数据后的行数据处理 """ new_row = {} for k, v in row_data.items(): field = self.serializer_fields.get(k) v = self.parse_value(field, v) new_row[k] = v return new_row def generate_data(self, fields_name, rows): data = [] for row in rows: # 空行不处理 if not any(row): continue row = self.load_row(row) row_data = dict(zip(fields_name, row)) row_data = self.process_row_data(row_data) data.append(row_data) return data @staticmethod def pop_help_text_if_need(rows): rows = list(rows) if not rows: return rows if rows[0][0] == '#Help': rows.pop(0) return rows def parse(self, stream, media_type=None, parser_context=None): assert parser_context is not None, '`parser_context` should not be `None`' view = parser_context['view'] request = view.request try: meta = request.META self.serializer_cls = view.get_serializer_class() self.serializer_fields = self.serializer_cls().fields except Exception as e: logger.debug(e, exc_info=True) raise ParseError('The resource does not support imports!') self.check_content_length(meta) try: stream_data = self.get_stream_data(stream) rows = self.generate_rows(stream_data) column_titles = self.get_column_titles(rows) field_names = self.convert_to_field_names(column_titles) # 给 `common.mixins.api.RenderToJsonMixin` 提供，暂时只能耦合 column_title_field_pairs = list(zip(column_titles, field_names)) column_title_field_pairs = [(k, v) for k, v in column_title_field_pairs if k and v] if not hasattr(request, 'jms_context'): request.jms_context = {} request.jms_context['column_title_field_pairs'] = column_title_field_pairs rows = self.pop_help_text_if_need(rows) data = self.generate_data(field_names, rows) return data except Exception as e: logger.error(e, exc_info=True) raise ParseError(_('Parse file error: {}').format(e))