@@ -31,6 +31,7 @@ import glob |
||
| 31 | 31 |
import io |
| 32 | 32 |
import os |
| 33 | 33 |
import re |
| 34 |
+import sys |
|
| 34 | 35 |
|
| 35 | 36 |
from django.core.management.base import BaseCommand, CommandError |
| 36 | 37 |
from django.db import transaction |
@@ -95,9 +96,31 @@ def parse_line(line): |
||
| 95 | 96 |
} |
| 96 | 97 |
|
| 97 | 98 |
|
| 99 |
+def iter_lines(path): |
|
| 100 |
+ """按 utf-8 读日志,失败则按 gbk 再试,最后兜底 latin-1(不抛异常)""" |
|
| 101 |
+ for encoding in ('utf-8', 'gbk', 'latin-1'):
|
|
| 102 |
+ try: |
|
| 103 |
+ handle = io.open(path, 'r', encoding=encoding, errors='strict') |
|
| 104 |
+ # 先读一小段确认能解开,避免读到一半才炸 |
|
| 105 |
+ head = handle.read(4096) |
|
| 106 |
+ handle.seek(0) |
|
| 107 |
+ except (UnicodeDecodeError, LookupError): |
|
| 108 |
+ try: |
|
| 109 |
+ handle.close() |
|
| 110 |
+ except Exception: |
|
| 111 |
+ pass |
|
| 112 |
+ continue |
|
| 113 |
+ |
|
| 114 |
+ for line in [head] + handle.readlines(): |
|
| 115 |
+ yield line |
|
| 116 |
+ |
|
| 117 |
+ handle.close() |
|
| 118 |
+ return |
|
| 119 |
+ |
|
| 120 |
+ |
|
| 98 | 121 |
def collect(log_glob): |
| 99 | 122 |
""" |
| 100 |
- 扫描日志,返回 ({(campaign_id, serial_no): {name, phone, timestamp}}, 扫描到的 join 条数)
|
|
| 123 |
+ 扫描日志,返回 ({(campaign_id, serial_no): {name, phone, timestamp}}, 文件数, 扫描到的 join 条数)
|
|
| 101 | 124 |
|
| 102 | 125 |
同一条名额出现多次时,优先保留带姓名的那次。 |
| 103 | 126 |
""" |
@@ -106,23 +129,35 @@ def collect(log_glob): |
||
| 106 | 129 |
scanned = 0 |
| 107 | 130 |
|
| 108 | 131 |
for path in files: |
| 109 |
- with io.open(path, 'r', encoding='utf-8', errors='ignore') as handle: |
|
| 110 |
- for line in handle: |
|
| 111 |
- record = parse_line(line) |
|
| 132 |
+ for line in iter_lines(path): |
|
| 133 |
+ record = parse_line(line) |
|
| 112 | 134 |
|
| 113 |
- if not record: |
|
| 114 |
- continue |
|
| 135 |
+ if not record: |
|
| 136 |
+ continue |
|
| 115 | 137 |
|
| 116 |
- scanned += 1 |
|
| 117 |
- key = (record['campaign_id'], record['serial_no']) |
|
| 118 |
- old = found.get(key) |
|
| 138 |
+ scanned += 1 |
|
| 139 |
+ key = (record['campaign_id'], record['serial_no']) |
|
| 140 |
+ old = found.get(key) |
|
| 119 | 141 |
|
| 120 |
- if old is None or (record['name'] and not old['name']): |
|
| 121 |
- found[key] = record |
|
| 142 |
+ if old is None or (record['name'] and not old['name']): |
|
| 143 |
+ found[key] = record |
|
| 122 | 144 |
|
| 123 | 145 |
return found, len(files), scanned |
| 124 | 146 |
|
| 125 | 147 |
|
| 148 |
+def setup_stdio_encoding(): |
|
| 149 |
+ """服务器 locale 可能是 POSIX,stdout 编码为 ascii,输出中文会直接抛异常""" |
|
| 150 |
+ for stream in ('stdout', 'stderr'):
|
|
| 151 |
+ handle = getattr(sys, stream, None) |
|
| 152 |
+ reconfigure = getattr(handle, 'reconfigure', None) |
|
| 153 |
+ |
|
| 154 |
+ if reconfigure: |
|
| 155 |
+ try: |
|
| 156 |
+ reconfigure(encoding='utf-8', errors='replace') |
|
| 157 |
+ except (ValueError, OSError): |
|
| 158 |
+ pass |
|
| 159 |
+ |
|
| 160 |
+ |
|
| 126 | 161 |
class Command(BaseCommand): |
| 127 | 162 |
help = '从 djlogit 日志回填抽奖名额的姓名/手机号(只填空值)' |
| 128 | 163 |
|
@@ -134,6 +169,8 @@ class Command(BaseCommand): |
||
| 134 | 169 |
parser.add_argument('--parse-only', default='', help='只解析日志并打印,不查数据库')
|
| 135 | 170 |
|
| 136 | 171 |
def handle(self, *args, **options): |
| 172 |
+ setup_stdio_encoding() |
|
| 173 |
+ |
|
| 137 | 174 |
log_glob = options['log_glob'] |
| 138 | 175 |
|
| 139 | 176 |
if options['parse_only']: |