import os import sys import time from datetime import datetime, timedelta from typing import Optional import click sys.path.insert(0, os.path.dirname(__file__)) from config import config from ingestor.novofon_api import NovofonApiClient from ingestor.downloader import download_audio from audio.preprocessor import convert_to_wav, split_stereo_channels from audio.stt import transcribe from aligner.aligner import align from storage.contacts import resolve_contacts, resolve_contact_name from storage.exporter import export_to_json from storage.directus import DirectusClient # ── Audio helpers ────────────────────────────────────────────────── def _get_wav_channels(wav_path: str) -> int: import wave try: with wave.open(wav_path, "rb") as wf: return wf.getnchannels() except Exception: return 2 def _is_mono_duplicate(wav_path: str, threshold: float = 0.98) -> bool: import wave import struct try: with wave.open(wav_path, "rb") as wf: if wf.getnchannels() != 2: return False sampwidth = wf.getsampwidth() framerate = wf.getframerate() nframes = wf.getnframes() total = min(nframes, framerate * 3) raw = wf.readframes(total) diffs = 0 non_silent = 0 for i in range(total): offset = i * 2 * sampwidth if sampwidth == 2: left = struct.unpack_from(" 100: diffs += 1 if non_silent == 0: return True return diffs / non_silent < (1 - threshold) except Exception: return False # ── Transcription pipeline ───────────────────────────────────────── def _transcribe_stereo(wav_path: str, meta: dict) -> dict: channels = _get_wav_channels(wav_path) if channels == 1: click.echo(" Mono file detected, single transcription.") left_segments, left_text = transcribe( wav_path, model_size=config.WHISPER_MODEL_SIZE, device=config.WHISPER_DEVICE, compute_type=config.WHISPER_COMPUTE_TYPE, ) final_segments = [] for seg in left_segments: final_segments.append({ "start": seg["start"], "end": seg["end"], "speaker": "unknown", "text": seg["text"].strip(), }) is_stereo = False elif _is_mono_duplicate(wav_path): click.echo(" Stereo with duplicate channels, treating as mono.") left_segments, left_text = transcribe( wav_path, model_size=config.WHISPER_MODEL_SIZE, device=config.WHISPER_DEVICE, compute_type=config.WHISPER_COMPUTE_TYPE, ) final_segments = [] for seg in left_segments: final_segments.append({ "start": seg["start"], "end": seg["end"], "speaker": "unknown", "text": seg["text"].strip(), }) is_stereo = False else: click.echo(" Real stereo, splitting channels...") left_wav, right_wav = split_stereo_channels(wav_path, config.TEMP_DIR) click.echo(" Transcribing left channel (callee)...") left_segments, left_text = transcribe( left_wav, model_size=config.WHISPER_MODEL_SIZE, device=config.WHISPER_DEVICE, compute_type=config.WHISPER_COMPUTE_TYPE, ) click.echo(" Transcribing right channel (caller)...") right_segments, right_text = transcribe( right_wav, model_size=config.WHISPER_MODEL_SIZE, device=config.WHISPER_DEVICE, compute_type=config.WHISPER_COMPUTE_TYPE, ) click.echo(" Aligning speakers & text...") final_segments = align(left_segments, right_segments) is_stereo = True caller_name = resolve_contact_name(meta.get("caller_number")) callee_name = resolve_contact_name(meta.get("callee_number")) caller_number = meta.get("caller_number") or "" callee_number = meta.get("callee_number") or "" full_text = "" for seg in final_segments: spk = seg["speaker"] if spk == "caller": label = caller_name or caller_number or "Звонящий" elif spk == "callee": label = callee_name or callee_number or "Принимающий" else: label = "?" full_text += f"[{label}]: {seg['text']}\n" contact_info = resolve_contacts( meta.get("caller_number"), meta.get("callee_number"), ) call_data = { **meta, **contact_info, "transcript": full_text, "segments": final_segments, "status": "processed", "raw_metadata": meta, } try: dc = DirectusClient() if dc.is_available(): collection = _choose_collection(meta) dc.create_call(call_data, collection=collection) click.echo(f" Saved to Directus ({collection}).") dc.close() except Exception as e: click.echo(f" Directus save error: {e}") to_remove = [wav_path] if is_stereo: to_remove.extend([left_wav, right_wav]) for f in to_remove: if f and os.path.exists(f): os.remove(f) return call_data def _process_call(audio_path: str, meta: dict) -> dict: click.echo(" Converting audio...") wav_path, channels = convert_to_wav(audio_path, config.TEMP_DIR) return _transcribe_stereo(wav_path, meta) def _get_audio_file(meta: dict, temp_dir: str) -> Optional[str]: url = meta.get("audio_url") if url: return download_audio(url, temp_dir) return None def _is_already_processed(call_id: str) -> bool: dc = DirectusClient() try: for col in ["calls", "calls_74951284933"]: resp = dc._client.get(f"/items/{col}", params={ "filter[call_id][_eq]": call_id, "limit": 1, "fields": "id", }) if resp.status_code == 200 and resp.json().get("data"): return True except Exception: pass finally: dc.close() return False def _choose_collection(meta: dict) -> str: caller = meta.get("caller_number") or "" callee = meta.get("callee_number") or "" for num in [caller, callee]: cleaned = num.replace("+", "").replace("-", "").replace(" ", "") if "74951284933" in cleaned: return "calls_74951284933" return "calls" # ── API batch processing ─────────────────────────────────────────── def _run_api_batch( limit: Optional[int] = None, date_from: Optional[str] = None, date_till: Optional[str] = None, ) -> int: api = NovofonApiClient() if not date_from: date_from = config.NOVOFON_DATE_FROM click.echo(f"Fetching calls from {date_from}...") calls = api.get_all_calls(date_from, date_till) api.close() click.echo(f"Found {len(calls)} calls via API.") tasks = [] for meta in calls: if _is_already_processed(meta.call_id): continue if not meta.audio_url: continue tasks.append(meta) if limit and len(tasks) >= limit: break click.echo(f"New calls with audio to process: {len(tasks)}") if not tasks: click.echo("Nothing new to process.") return 0 click.echo(f"\n=== Download + convert ({len(tasks)} files) ===") prepared = [] for meta in tasks: try: click.echo(f" Preparing {meta.call_id}...") audio_path = _get_audio_file( {"audio_url": meta.audio_url}, config.TEMP_DIR, ) if not audio_path: click.echo(" No audio, skipping.") continue wav_path, _ = convert_to_wav(audio_path, config.TEMP_DIR) if audio_path != wav_path and os.path.exists(audio_path): os.remove(audio_path) prepared.append((meta, wav_path)) except Exception as e: click.echo(f" Error: {e}") if not prepared: click.echo("No files prepared for transcription.") return 0 click.echo(f"\n=== Transcribe + save ({len(prepared)} files, sequential on GPU) ===") processed = 0 for meta, wav_path in prepared: try: call_data = _transcribe_stereo(wav_path, { "call_id": meta.call_id, "datetime": meta.datetime, "direction": meta.direction, "caller_number": meta.caller_number, "callee_number": meta.callee_number, "employee": meta.employee, "duration_sec": meta.duration_sec, "audio_url": meta.audio_url, }) processed += 1 click.echo(f" Done! Call ID: {call_data.get('call_id')}") except Exception as e: click.echo(f" Error processing {meta.call_id}: {e}") if wav_path and os.path.exists(wav_path): os.remove(wav_path) return processed # ── CLI ──────────────────────────────────────────────────────────── @click.group() def cli(): pass @cli.command() @click.option("--limit", default=None, type=int, help="Max calls to process") @click.option("--date-from", default=None, help="YYYY-MM-DD or YYYY-MM-DD hh:mm:ss") def process(limit, date_from): cnt = _run_api_batch(limit=limit, date_from=date_from) click.echo(f"\nProcessed {cnt} calls.") @cli.command() @click.option("--file", required=True, help="Path to audio file") @click.option("--caller", default=None, help="Caller number") @click.option("--callee", default=None, help="Callee number") @click.option("--duration", default=None, type=int, help="Duration in seconds") def process_file(file, caller, callee, duration): click.echo(f"Processing file: {file}") meta = { "call_id": f"manual_{os.path.splitext(os.path.basename(file))[0]}", "caller_number": caller, "callee_number": callee, "direction": "inbound" if caller else None, "duration_sec": duration, "audio_url": file, } call_data = _process_call(file, meta) click.echo(f"Done! Call ID: {call_data.get('call_id')}") @cli.command() @click.option("--output", default="export.json", help="Output file path") def export(output): export_to_json(output) click.echo(f"Exported to {output}") @cli.command() @click.option("--max", "max_per_cycle", default=None, type=int, help="Max calls per cycle") @click.option("--hours-back", default=6, type=int, help="How far back to check for calls") def daemon(max_per_cycle, hours_back): import schedule max_calls = max_per_cycle or config.MAX_PER_CYCLE def job(): now = datetime.utcnow() since = (now - timedelta(hours=hours_back)).strftime("%Y-%m-%d %H:%M:%S") till = now.strftime("%Y-%m-%d %H:%M:%S") click.echo(f"Checking for new calls since {since}...") try: cnt = _run_api_batch(limit=max_calls, date_from=since, date_till=till) click.echo(f"Processed {cnt} calls this cycle.") except Exception as e: click.echo(f"Error: {e}") schedule.every(5).minutes.do(job) click.echo(f"Daemon started. Checking every 5 minutes, max {max_calls} calls/cycle.") while True: schedule.run_pending() time.sleep(30) if __name__ == "__main__": cli()