fix: 回填命令适配服务器 locale 与日志编码

- 输出前把 stdout/stderr 切到 utf-8,服务器 locale 为 POSIX 时写中文会抛编码异常
- 读日志按 utf-8 → gbk → latin-1 依次回退,不再因单个文件编码不同而中断

FFIB vor 4 Stunden
Ursprung
Commit
b0b0ffa4be
1 geänderte Dateien mit 48 neuen Zeilen und 11 gelöschten Zeilen
  1. 48 11
      sales/management/commands/backfill_lottery_names.py

+ 48 - 11
sales/management/commands/backfill_lottery_names.py

@@ -31,6 +31,7 @@ import glob
31 31
 import io
32 32
 import os
33 33
 import re
34
+import sys
34 35
 
35 36
 from django.core.management.base import BaseCommand, CommandError
36 37
 from django.db import transaction
@@ -95,9 +96,31 @@ def parse_line(line):
95 96
     }
96 97
 
97 98
 
99
+def iter_lines(path):
100
+    """按 utf-8 读日志,失败则按 gbk 再试,最后兜底 latin-1(不抛异常)"""
101
+    for encoding in ('utf-8', 'gbk', 'latin-1'):
102
+        try:
103
+            handle = io.open(path, 'r', encoding=encoding, errors='strict')
104
+            # 先读一小段确认能解开,避免读到一半才炸
105
+            head = handle.read(4096)
106
+            handle.seek(0)
107
+        except (UnicodeDecodeError, LookupError):
108
+            try:
109
+                handle.close()
110
+            except Exception:
111
+                pass
112
+            continue
113
+
114
+        for line in [head] + handle.readlines():
115
+            yield line
116
+
117
+        handle.close()
118
+        return
119
+
120
+
98 121
 def collect(log_glob):
99 122
     """
100
-    扫描日志,返回 ({(campaign_id, serial_no): {name, phone, timestamp}}, 扫描到的 join 条数)
123
+    扫描日志,返回 ({(campaign_id, serial_no): {name, phone, timestamp}}, 文件数, 扫描到的 join 条数)
101 124
 
102 125
     同一条名额出现多次时,优先保留带姓名的那次。
103 126
     """
@@ -106,23 +129,35 @@ def collect(log_glob):
106 129
     scanned = 0
107 130
 
108 131
     for path in files:
109
-        with io.open(path, 'r', encoding='utf-8', errors='ignore') as handle:
110
-            for line in handle:
111
-                record = parse_line(line)
132
+        for line in iter_lines(path):
133
+            record = parse_line(line)
112 134
 
113
-                if not record:
114
-                    continue
135
+            if not record:
136
+                continue
115 137
 
116
-                scanned += 1
117
-                key = (record['campaign_id'], record['serial_no'])
118
-                old = found.get(key)
138
+            scanned += 1
139
+            key = (record['campaign_id'], record['serial_no'])
140
+            old = found.get(key)
119 141
 
120
-                if old is None or (record['name'] and not old['name']):
121
-                    found[key] = record
142
+            if old is None or (record['name'] and not old['name']):
143
+                found[key] = record
122 144
 
123 145
     return found, len(files), scanned
124 146
 
125 147
 
148
+def setup_stdio_encoding():
149
+    """服务器 locale 可能是 POSIX,stdout 编码为 ascii,输出中文会直接抛异常"""
150
+    for stream in ('stdout', 'stderr'):
151
+        handle = getattr(sys, stream, None)
152
+        reconfigure = getattr(handle, 'reconfigure', None)
153
+
154
+        if reconfigure:
155
+            try:
156
+                reconfigure(encoding='utf-8', errors='replace')
157
+            except (ValueError, OSError):
158
+                pass
159
+
160
+
126 161
 class Command(BaseCommand):
127 162
     help = '从 djlogit 日志回填抽奖名额的姓名/手机号(只填空值)'
128 163
 
@@ -134,6 +169,8 @@ class Command(BaseCommand):
134 169
         parser.add_argument('--parse-only', default='', help='只解析日志并打印,不查数据库')
135 170
 
136 171
     def handle(self, *args, **options):
172
+        setup_stdio_encoding()
173
+
137 174
         log_glob = options['log_glob']
138 175
 
139 176
         if options['parse_only']: