@@ -1,294 +0,0 @@ |
||
| 1 |
-# -*- coding: utf-8 -*- |
|
| 2 |
- |
|
| 3 |
-""" |
|
| 4 |
-把抽奖名额里丢失的姓名 / 手机号,从服务器日志里回填。 |
|
| 5 |
- |
|
| 6 |
-背景 |
|
| 7 |
----- |
|
| 8 |
-服务端曾只读 POST 的 'name',而小程序传的字段叫 'title',导致 2026-11-22 |
|
| 9 |
-修复上线之前产生的名额 name 为空(phone 正常,字段名一致)。djlogit 把每次 |
|
| 10 |
-请求的 POST 参数都记在日志里,可以按 (campaign_id, serial_no) 把姓名找回来。 |
|
| 11 |
- |
|
| 12 |
-用法 |
|
| 13 |
----- |
|
| 14 |
- # 只看能补多少,不动数据库 |
|
| 15 |
- python manage.py backfill_lottery_names --dry-run |
|
| 16 |
- |
|
| 17 |
- # 只处理某个活动 |
|
| 18 |
- python manage.py backfill_lottery_names --dry-run --campaign-id 4SRByFKXeSqH7VfUp5D9Hn |
|
| 19 |
- |
|
| 20 |
- # 确认数字没问题后落库(只填空值,不覆盖已有数据) |
|
| 21 |
- python manage.py backfill_lottery_names --apply |
|
| 22 |
- |
|
| 23 |
- # 只解析日志不查库(排查用) |
|
| 24 |
- python manage.py backfill_lottery_names --parse-only /tmp/kodotamron.logit.log |
|
| 25 |
-""" |
|
| 26 |
- |
|
| 27 |
-from __future__ import print_function, unicode_literals |
|
| 28 |
- |
|
| 29 |
-import ast |
|
| 30 |
-import glob |
|
| 31 |
-import io |
|
| 32 |
-import os |
|
| 33 |
-import re |
|
| 34 |
-import sys |
|
| 35 |
- |
|
| 36 |
-from django.core.management.base import BaseCommand, CommandError |
|
| 37 |
-from django.db import transaction |
|
| 38 |
- |
|
| 39 |
-from sales.models import LotteryEntryInfo |
|
| 40 |
- |
|
| 41 |
-JOIN_MARKER = 'func=lottery_join_api' |
|
| 42 |
-POST_MARKER = 'flag=post' |
|
| 43 |
-TIME_RE = re.compile(r'^(?:DEBUG|INFO|WARNING|ERROR)\s+(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})')
|
|
| 44 |
- |
|
| 45 |
-DEFAULT_LOG_GLOB = '/tmp/kodotamron.logit.log*' |
|
| 46 |
- |
|
| 47 |
- |
|
| 48 |
-def parse_content(raw): |
|
| 49 |
- """把 '<QueryDict: {u'a': [u'b']}>' 解析成普通 dict,解析失败返回 None"""
|
|
| 50 |
- text = raw.strip() |
|
| 51 |
- start = text.find('<QueryDict:')
|
|
| 52 |
- |
|
| 53 |
- if start < 0: |
|
| 54 |
- return None |
|
| 55 |
- |
|
| 56 |
- text = text[start + len('<QueryDict:'):].strip()
|
|
| 57 |
- |
|
| 58 |
- if text.endswith('>'):
|
|
| 59 |
- text = text[:-1].strip() |
|
| 60 |
- |
|
| 61 |
- try: |
|
| 62 |
- value = ast.literal_eval(text) |
|
| 63 |
- except (ValueError, SyntaxError): |
|
| 64 |
- return None |
|
| 65 |
- |
|
| 66 |
- if not isinstance(value, dict): |
|
| 67 |
- return None |
|
| 68 |
- |
|
| 69 |
- # 每个 key 取第一个值 |
|
| 70 |
- return {k: (v[0] if isinstance(v, list) and v else v) for k, v in value.items()}
|
|
| 71 |
- |
|
| 72 |
- |
|
| 73 |
-def parse_line(line): |
|
| 74 |
- """解析一行日志;不是 lottery_join_api 的 post 记录返回 None""" |
|
| 75 |
- if JOIN_MARKER not in line or POST_MARKER not in line: |
|
| 76 |
- return None |
|
| 77 |
- |
|
| 78 |
- index = line.find('content=')
|
|
| 79 |
- |
|
| 80 |
- if index < 0: |
|
| 81 |
- return None |
|
| 82 |
- |
|
| 83 |
- params = parse_content(line[index + len('content='):])
|
|
| 84 |
- |
|
| 85 |
- if not params: |
|
| 86 |
- return None |
|
| 87 |
- |
|
| 88 |
- matched = TIME_RE.match(line) |
|
| 89 |
- |
|
| 90 |
- return {
|
|
| 91 |
- 'campaign_id': (params.get('campaign_id') or '').strip(),
|
|
| 92 |
- 'serial_no': (params.get('serial_no') or '').strip(),
|
|
| 93 |
- 'name': (params.get('name') or params.get('title') or '').strip(),
|
|
| 94 |
- 'phone': (params.get('phone') or '').strip(),
|
|
| 95 |
- 'timestamp': matched.group(1) if matched else '', |
|
| 96 |
- } |
|
| 97 |
- |
|
| 98 |
- |
|
| 99 |
-def iter_lines(path): |
|
| 100 |
- """ |
|
| 101 |
- 逐行读日志。 |
|
| 102 |
- |
|
| 103 |
- djlogit 的记录里,中文是 \u4e54 这种转义形式(纯 ASCII), |
|
| 104 |
- 文件名和内容都按 utf-8 读即可;errors="replace" 兜住个别脏字节, |
|
| 105 |
- 避免某个归档文件里混进坏数据就整批中断。 |
|
| 106 |
- """ |
|
| 107 |
- with io.open(path, 'r', encoding='utf-8', errors='replace') as handle: |
|
| 108 |
- for line in handle: |
|
| 109 |
- yield line |
|
| 110 |
- |
|
| 111 |
- |
|
| 112 |
-def collect(log_glob): |
|
| 113 |
- """ |
|
| 114 |
- 扫描日志,返回 ({(campaign_id, serial_no): {name, phone, timestamp}}, 文件数, 扫描到的 join 条数)
|
|
| 115 |
- |
|
| 116 |
- 同一条名额出现多次时,优先保留带姓名的那次。 |
|
| 117 |
- """ |
|
| 118 |
- found = {}
|
|
| 119 |
- files = sorted(glob.glob(log_glob)) |
|
| 120 |
- scanned = 0 |
|
| 121 |
- |
|
| 122 |
- for path in files: |
|
| 123 |
- for line in iter_lines(path): |
|
| 124 |
- record = parse_line(line) |
|
| 125 |
- |
|
| 126 |
- if not record: |
|
| 127 |
- continue |
|
| 128 |
- |
|
| 129 |
- scanned += 1 |
|
| 130 |
- key = (record['campaign_id'], record['serial_no']) |
|
| 131 |
- old = found.get(key) |
|
| 132 |
- |
|
| 133 |
- if old is None or (record['name'] and not old['name']): |
|
| 134 |
- found[key] = record |
|
| 135 |
- |
|
| 136 |
- return found, len(files), scanned |
|
| 137 |
- |
|
| 138 |
- |
|
| 139 |
-PY2 = sys.version_info[0] == 2 |
|
| 140 |
- |
|
| 141 |
- |
|
| 142 |
-def emit(stream, text): |
|
| 143 |
- """ |
|
| 144 |
- 统一输出,绕过 Django 的 OutputWrapper。 |
|
| 145 |
- |
|
| 146 |
- py2 + Django 1.x 下 OutputWrapper.write 会做 msg.endswith(unicode '\n'): |
|
| 147 |
- - 传 utf-8 字节 → bytes.endswith(unicode) 触发 ascii 解码 → UnicodeDecodeError |
|
| 148 |
- - 传 unicode → sys.stdout 编码是 ascii 时 → UnicodeEncodeError |
|
| 149 |
- 所以直接写底层流:py2 写 utf-8 字节,py3 写 str。 |
|
| 150 |
- """ |
|
| 151 |
- if PY2: |
|
| 152 |
- target = getattr(stream, '_out', None) or stream |
|
| 153 |
- target.write(text.encode('utf-8'))
|
|
| 154 |
- target.flush() |
|
| 155 |
- else: |
|
| 156 |
- stream.write(text) |
|
| 157 |
- |
|
| 158 |
- |
|
| 159 |
-def dump_ascii_traceback(stream): |
|
| 160 |
- """ |
|
| 161 |
- 把当前异常栈转成纯 ASCII 再输出。 |
|
| 162 |
- |
|
| 163 |
- py2 环境下 stderr 常见编码是 ascii,Django 打印含中文的堆栈时会二次抛错, |
|
| 164 |
- 结果只剩一行 UnicodeDecodeError,看不到真正出错的位置。 |
|
| 165 |
- """ |
|
| 166 |
- import traceback |
|
| 167 |
- |
|
| 168 |
- text = traceback.format_exc() |
|
| 169 |
- |
|
| 170 |
- if not isinstance(text, type(u'')): |
|
| 171 |
- text = text.decode('utf-8', 'replace')
|
|
| 172 |
- |
|
| 173 |
- emit(stream, text.encode('ascii', 'backslashreplace').decode('ascii'))
|
|
| 174 |
- |
|
| 175 |
- |
|
| 176 |
-def setup_stdio_encoding(): |
|
| 177 |
- """服务器 locale 可能是 POSIX,stdout 编码为 ascii,输出中文会直接抛异常""" |
|
| 178 |
- for stream in ('stdout', 'stderr'):
|
|
| 179 |
- handle = getattr(sys, stream, None) |
|
| 180 |
- reconfigure = getattr(handle, 'reconfigure', None) |
|
| 181 |
- |
|
| 182 |
- if reconfigure: |
|
| 183 |
- try: |
|
| 184 |
- reconfigure(encoding='utf-8', errors='replace') |
|
| 185 |
- except (ValueError, OSError): |
|
| 186 |
- pass |
|
| 187 |
- |
|
| 188 |
- |
|
| 189 |
-class Command(BaseCommand): |
|
| 190 |
- help = '从 djlogit 日志回填抽奖名额的姓名/手机号(只填空值)' |
|
| 191 |
- |
|
| 192 |
- def add_arguments(self, parser): |
|
| 193 |
- parser.add_argument('--log-glob', default=DEFAULT_LOG_GLOB, help='日志路径通配符')
|
|
| 194 |
- parser.add_argument('--campaign-id', default='', help='只处理指定活动')
|
|
| 195 |
- parser.add_argument('--dry-run', action='store_true', help='只统计,不写库')
|
|
| 196 |
- parser.add_argument('--apply', action='store_true', help='真正写库')
|
|
| 197 |
- parser.add_argument('--parse-only', default='', help='只解析日志并打印,不查数据库')
|
|
| 198 |
- |
|
| 199 |
- def handle(self, *args, **options): |
|
| 200 |
- setup_stdio_encoding() |
|
| 201 |
- |
|
| 202 |
- try: |
|
| 203 |
- self.run_command(*args, **options) |
|
| 204 |
- except Exception: |
|
| 205 |
- dump_ascii_traceback(self.stderr) |
|
| 206 |
- raise CommandError('backfill failed, see ASCII traceback above')
|
|
| 207 |
- |
|
| 208 |
- def run_command(self, *args, **options): |
|
| 209 |
- log_glob = options['log_glob'] |
|
| 210 |
- |
|
| 211 |
- if options['parse_only']: |
|
| 212 |
- return self.handle_parse_only(options['parse_only']) |
|
| 213 |
- |
|
| 214 |
- if not options['dry_run'] and not options['apply']: |
|
| 215 |
- raise CommandError('specify either --dry-run (count only) or --apply (write to db)')
|
|
| 216 |
- |
|
| 217 |
- found, file_count, scanned = collect(log_glob) |
|
| 218 |
- |
|
| 219 |
- emit(self.stdout, '日志文件: %d 个,解析到 join 请求: %d 条,可回填组合: %d 个' % ( |
|
| 220 |
- file_count, scanned, len(found))) |
|
| 221 |
- |
|
| 222 |
- if not file_count: |
|
| 223 |
- emit(self.stdout, self.style.WARNING('没有匹配到日志文件:%s' % log_glob))
|
|
| 224 |
- |
|
| 225 |
- entries = LotteryEntryInfo.objects.filter(status=True) |
|
| 226 |
- |
|
| 227 |
- if options['campaign_id']: |
|
| 228 |
- entries = entries.filter(campaign_id=options['campaign_id']) |
|
| 229 |
- |
|
| 230 |
- missing = [entry for entry in entries.only('id', 'campaign_id', 'serial_no', 'name', 'phone')
|
|
| 231 |
- if not (entry.name or '').strip()] |
|
| 232 |
- |
|
| 233 |
- emit(self.stdout, '待检查名额: %d 条,其中姓名为空: %d 条' % (entries.count(), len(missing))) |
|
| 234 |
- |
|
| 235 |
- todo = [] |
|
| 236 |
- |
|
| 237 |
- for entry in missing: |
|
| 238 |
- record = found.get((entry.campaign_id, entry.serial_no)) |
|
| 239 |
- |
|
| 240 |
- if not record or not record['name']: |
|
| 241 |
- continue |
|
| 242 |
- |
|
| 243 |
- todo.append((entry, record)) |
|
| 244 |
- |
|
| 245 |
- emit(self.stdout, '日志里能匹配到且带姓名的: %d 条' % len(todo)) |
|
| 246 |
- emit(self.stdout, '补不上的: %d 条(对应日志已被清理)' % (len(missing) - len(todo))) |
|
| 247 |
- |
|
| 248 |
- for entry, record in todo[:10]: |
|
| 249 |
- will_fill_phone = record['phone'] and not (entry.phone or '').strip() |
|
| 250 |
- emit(self.stdout, ' %s %s → %s%s' % ( |
|
| 251 |
- entry.campaign_id[:8], |
|
| 252 |
- entry.serial_no, |
|
| 253 |
- record['name'], |
|
| 254 |
- '(含手机号)' if will_fill_phone else '', |
|
| 255 |
- )) |
|
| 256 |
- |
|
| 257 |
- if len(todo) > 10: |
|
| 258 |
- emit(self.stdout, ' ... 其余 %d 条省略' % (len(todo) - 10)) |
|
| 259 |
- |
|
| 260 |
- if options['dry_run']: |
|
| 261 |
- emit(self.stdout, self.style.SUCCESS('\n[dry-run] 未写库。确认后加 --apply 执行。'))
|
|
| 262 |
- return |
|
| 263 |
- |
|
| 264 |
- updated = 0 |
|
| 265 |
- |
|
| 266 |
- with transaction.atomic(): |
|
| 267 |
- for entry, record in todo: |
|
| 268 |
- update_fields = [] |
|
| 269 |
- |
|
| 270 |
- if not (entry.name or '').strip(): |
|
| 271 |
- entry.name = record['name'] |
|
| 272 |
- update_fields.append('name')
|
|
| 273 |
- |
|
| 274 |
- if not (entry.phone or '').strip() and record['phone']: |
|
| 275 |
- entry.phone = record['phone'] |
|
| 276 |
- update_fields.append('phone')
|
|
| 277 |
- |
|
| 278 |
- if update_fields: |
|
| 279 |
- entry.save(update_fields=update_fields) |
|
| 280 |
- updated += 1 |
|
| 281 |
- |
|
| 282 |
- emit(self.stdout, self.style.SUCCESS('\n已更新 %d 条名额' % updated))
|
|
| 283 |
- |
|
| 284 |
- def handle_parse_only(self, path): |
|
| 285 |
- if not os.path.exists(path): |
|
| 286 |
- raise CommandError('log file not found: %s' % path)
|
|
| 287 |
- |
|
| 288 |
- found, file_count, scanned = collect(path) |
|
| 289 |
- |
|
| 290 |
- emit(self.stdout, '文件: %d 个,join 请求: %d 条,可回填组合: %d 个' % (file_count, scanned, len(found))) |
|
| 291 |
- |
|
| 292 |
- for (campaign_id, serial_no), record in list(found.items())[:10]: |
|
| 293 |
- emit(self.stdout, ' %s / %s → %s %s' % ( |
|
| 294 |
- campaign_id[:8], serial_no, record['name'] or '(空)', record['phone'])) |