#!/usr/bin/env python2
"""Inspect pitch movement at pause-delimited utterance endings.

Python 2.7 standard library only. Input must be mono, 16-bit PCM WAV.
"""

from __future__ import division, print_function

import argparse
import array
import csv
import json
import math
import os
import re
import sys
import wave


def nearest(value):
    return int(math.floor(value + 0.5))


def median(values):
    ordered = sorted(values)
    middle = len(ordered) // 2
    if len(ordered) % 2:
        return float(ordered[middle])
    return (ordered[middle - 1] + ordered[middle]) / 2.0


def percentile(values, fraction):
    ordered = sorted(values)
    return ordered[int((len(ordered) - 1) * fraction)]


def read_wav(path):
    source = wave.open(path, "rb")
    try:
        if (source.getnchannels() != 1 or source.getsampwidth() != 2 or
                source.getcomptype() != "NONE"):
            raise ValueError("input must be mono, 16-bit PCM WAV")
        sample_rate = source.getframerate()
        raw = source.readframes(source.getnframes())
    finally:
        source.close()
    samples = array.array("h")
    if hasattr(samples, "frombytes"):
        samples.frombytes(raw)
    else:
        samples.fromstring(raw)
    if sys.byteorder != "little":
        samples.byteswap()
    return samples, sample_rate, raw


def energy_frames(samples, sample_rate):
    window = max(1, nearest(0.020 * sample_rate))
    hop = max(1, nearest(0.010 * sample_rate))
    frames = []
    for start in range(0, len(samples), hop):
        part = samples[start:min(start + window, len(samples))]
        if not part:
            break
        power = sum(value * value for value in part) / float(len(part))
        frames.append({"start": start / float(sample_rate),
                       "end": (start + len(part)) / float(sample_rate),
                       "time": (start + len(part) / 2.0) / sample_rate,
                       "rms": math.sqrt(power) / 32768.0})
    return frames


def fill_short_gaps(active, max_gap):
    result = list(active)
    pos = 0
    while pos < len(result):
        if result[pos]:
            pos += 1
            continue
        end = pos
        while end < len(result) and not result[end]:
            end += 1
        if pos > 0 and end < len(result) and end - pos <= max_gap:
            result[pos:end] = [True] * (end - pos)
        pos = end
    return result


def speech_regions(frames, duration, min_pause, min_speech):
    if not frames:
        return [], None
    levels = [frame["rms"] for frame in frames]
    high = max(percentile(levels, 0.95), max(levels) * 0.10)
    low = percentile(levels, 0.10)
    if high < 0.0015:
        return [], None
    if low > high * 0.25:
        threshold = max(0.0015, high * 0.08)
    else:
        threshold = max(0.0015, high * 0.04, low * 2.0)
    active = fill_short_gaps([x >= threshold for x in levels], 8)
    runs = []
    pos = 0
    while pos < len(active):
        if not active[pos]:
            pos += 1
            continue
        end = pos
        while end < len(active) and active[end]:
            end += 1
        start_time = frames[pos]["start"]
        end_time = min(duration, frames[end - 1]["end"])
        if end_time - start_time >= min_speech:
            runs.append([start_time, end_time])
        pos = end
    merged = []
    for start, end in runs:
        if merged and start - merged[-1][1] < min_pause:
            merged[-1][1] = end
        else:
            merged.append([start, end])
    return merged, threshold


def downsample(samples, sample_rate):
    factor = max(1, sample_rate // 4000)
    reduced = array.array("f")
    for start in range(0, len(samples), factor):
        part = samples[start:start + factor]
        reduced.append(sum(part) / float(len(part)))
    return reduced, sample_rate / float(factor)


def estimate_f0(part, sample_rate, floor, ceiling, previous):
    count = len(part)
    mean = sum(part) / float(count)
    windowed = [(part[i] - mean) * (0.5 - 0.5 * math.cos(
        2.0 * math.pi * i / (count - 1))) for i in range(count)]
    squares = [0.0]
    for value in windowed:
        squares.append(squares[-1] + value * value)
    if squares[-1] / count < (0.0015 * 32768.0) ** 2:
        return None, 0.0
    first_lag = max(2, int(math.ceil(sample_rate / ceiling)))
    last_lag = min(count // 2, int(sample_rate / floor))
    if first_lag >= last_lag:
        return None, 0.0
    candidates = []
    for lag in range(first_lag, last_lag + 1):
        dot = 0.0
        for i in range(count - lag):
            dot += windowed[i] * windowed[i + lag]
        left_energy = squares[count - lag]
        right_energy = squares[count] - squares[lag]
        strength = dot / math.sqrt(max(left_energy * right_energy, 1e-12))
        candidates.append((lag, strength))
    peaks = []
    for index, candidate in enumerate(candidates):
        before = candidates[index - 1][1] if index else -1.0
        after = candidates[index + 1][1] if index + 1 < len(candidates) else -1.0
        if 0 < index < len(candidates) - 1 and candidate[1] >= before and candidate[1] >= after:
            lag = float(candidate[0])
            strength = candidate[1]
            curvature = before - 2.0 * strength + after
            if curvature < 0:
                shift = 0.5 * (before - after) / curvature
                lag += shift
                strength -= 0.25 * (before - after) * shift
            peaks.append((lag, strength))
    best = None
    best_score = -1e9
    for lag, strength in peaks:
        if strength < 0.55:
            continue
        frequency = sample_rate / lag
        score = strength - 0.015 * math.log(lag / float(first_lag), 2)
        if previous is not None:
            score -= 0.04 * abs(math.log(frequency / previous, 2))
        if score > best_score:
            best = (frequency, strength)
            best_score = score
    return best if best is not None else (None, 0.0)


def pitch_track(samples, sample_rate, start, end, floor, ceiling):
    window = max(4, nearest(0.060 * sample_rate))
    half = window // 2
    hop_time = 0.010
    first = max(start + half / sample_rate, end - 1.0)
    last = end - half / sample_rate
    result = []
    previous = None
    unvoiced_frames = 0
    step = 0
    while first + step * hop_time <= last + 1e-9:
        time = first + step * hop_time
        center = nearest(time * sample_rate)
        part = samples[center - half:center - half + window]
        if len(part) != window:
            break
        f0, strength = estimate_f0(part, sample_rate, floor, ceiling, previous)
        result.append({"time": round(time, 4), "f0": round(f0, 2) if f0 else None,
                       "strength": round(strength, 3)})
        if f0 is not None:
            previous = f0
            unvoiced_frames = 0
        else:
            unvoiced_frames += 1
            if unvoiced_frames >= 3:
                previous = None
        step += 1
    return result


def summarize(track, end):
    tail = [point for point in track if point["time"] >= end - 0.4]
    voiced = [point for point in tail if point["f0"] is not None]
    coverage = len(voiced) / float(len(tail)) if tail else 0.0
    result = {"shape": "unknown", "terminal_change_st": None,
              "voiced_fraction": round(coverage, 3)}
    if (len(voiced) < 8 or voiced[-1]["time"] - voiced[0]["time"] < 0.12 or
            end - voiced[-1]["time"] > 0.20):
        return result
    third = max(2, len(voiced) // 3)
    early = median([point["f0"] for point in voiced[:third]])
    middle = median([point["f0"] for point in voiced[third:-third]])
    late = median([point["f0"] for point in voiced[-third:]])
    change = 12.0 * math.log(late / early, 2)
    first_leg = 12.0 * math.log(middle / early, 2)
    second_leg = 12.0 * math.log(late / middle, 2)
    if first_leg <= -1.0 and second_leg >= 1.0:
        shape = "fall-rise"
    elif change >= 1.25:
        shape = "rise"
    elif change <= -1.25:
        shape = "fall"
    else:
        shape = "level"
    result["shape"] = shape
    result["terminal_change_st"] = round(change, 2)
    return result


def write_clip(path, raw, sample_rate, start, end):
    left = max(0, int(start * sample_rate)) * 2
    right = max(0, int(end * sample_rate)) * 2
    destination = wave.open(path, "wb")
    try:
        destination.setnchannels(1)
        destination.setsampwidth(2)
        destination.setframerate(sample_rate)
        destination.writeframes(raw[left:right])
    finally:
        destination.close()


def svg_chart(path, frames, track, region, shape):
    start, end = region
    plot_start = max(start, end - 1.0)
    left, right = 55, 730
    energy_top, energy_bottom = 42, 105
    pitch_top, pitch_bottom = 150, 235
    span = max(end - plot_start, 0.001)
    def x_at(time):
        return left + (time - plot_start) / span * (right - left)
    energy = [point for point in frames if plot_start <= point["time"] <= end]
    maximum = max([point["rms"] for point in energy] or [1e-9])
    energy_points = " ".join("%.1f,%.1f" % (
        x_at(point["time"]), energy_bottom - point["rms"] / maximum *
        (energy_bottom - energy_top)) for point in energy)
    valid = [point for point in track if point["f0"] is not None]
    reference = median([point["f0"] for point in valid]) if valid else 150.0
    semitones = [12.0 * math.log(point["f0"] / reference, 2) for point in valid]
    low = min(semitones + [-2.0]) - 1.0
    high = max(semitones + [2.0]) + 1.0
    def y_at(f0):
        value = 12.0 * math.log(f0 / reference, 2)
        return pitch_bottom - (value - low) / (high - low) * (pitch_bottom - pitch_top)
    paths = []
    run = []
    for point in track + [{"f0": None}]:
        if point["f0"] is None:
            if run:
                paths.append(" ".join(run))
                run = []
        else:
            run.append("%.1f,%.1f" % (x_at(point["time"]), y_at(point["f0"])))
    parts = [
        '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 760 275">',
        '<rect width="760" height="275" fill="white"/>',
        '<g fill="#333" font-family="sans-serif" font-size="13">',
        '<text x="55" y="22">Ending at %.2fs - %s</text>' % (end, shape),
        '<text x="8" y="73">RMS</text>',
        '<text x="8" y="191">F0</text>',
        '<text x="55" y="260">%.2fs</text>' % plot_start,
        '<text x="690" y="260">%.2fs</text>' % end,
        '</g>',
        '<path d="M55 105 H730 M55 235 H730" stroke="#ddd"/>',
        '<polyline points="%s" fill="none" stroke="#666" stroke-width="1.4"/>' % energy_points,
    ]
    for points in paths:
        parts.append('<polyline points="%s" fill="none" stroke="#1267a8" stroke-width="2"/>' % points)
    if not valid:
        parts.append('<text x="55" y="190" fill="#777" font-family="sans-serif" font-size="13">No reliable voiced frames</text>')
    parts.append('</svg>')
    with open(path, "wb") as destination:
        destination.write("\n".join(parts).encode("utf-8"))


def analyze(path, output, floor, ceiling, min_pause, min_speech):
    samples, sample_rate, raw = read_wav(path)
    if sample_rate < 2000:
        raise ValueError("sample rate must be at least 2000 Hz")
    duration = len(samples) / float(sample_rate)
    frames = energy_frames(samples, sample_rate)
    regions, threshold = speech_regions(frames, duration, min_pause, min_speech)
    reduced, reduced_rate = downsample(samples, sample_rate)
    rows = []
    for number, region in enumerate(regions, 1):
        start, end = region
        following = regions[number][0] - end if number < len(regions) else duration - end
        track = pitch_track(reduced, reduced_rate, start, end, floor, ceiling)
        summary = summarize(track, end)
        basename = "ending_%03d" % number
        clip = basename + ".wav"
        chart = basename + ".svg"
        clip_start = max(start, end - 1.5)
        clip_end = min(duration, end + 0.20)
        write_clip(os.path.join(output, clip), raw, sample_rate, clip_start, clip_end)
        svg_chart(os.path.join(output, chart), frames, track, region, summary["shape"])
        row = {"index": number, "start_s": round(start, 3),
               "end_s": round(end, 3), "following_pause_s": round(following, 3),
               "clip_start_s": round(clip_start, 3), "clip": clip, "chart": chart,
               "track": track}
        row.update(summary)
        rows.append(row)
    fields = ["index", "start_s", "end_s", "following_pause_s",
              "terminal_change_st", "shape", "voiced_fraction", "clip", "chart"]
    if sys.version_info[0] < 3:
        destination = open(os.path.join(output, "endings.csv"), "wb")
    else:
        destination = open(os.path.join(output, "endings.csv"), "w", newline="")
    try:
        writer = csv.writer(destination)
        writer.writerow(fields)
        for row in rows:
            writer.writerow([row[field] for field in fields])
    finally:
        destination.close()
    report = {"input": os.path.abspath(path), "sample_rate": sample_rate,
              "duration_s": round(duration, 3), "energy_threshold": threshold,
              "pitch_floor_hz": floor, "pitch_ceiling_hz": ceiling,
              "min_pause_s": min_pause, "endings": rows}
    with open(os.path.join(output, "endings.json"), "wb") as destination:
        destination.write(json.dumps(report, indent=2, sort_keys=True).encode("utf-8"))
    current = set([row["clip"] for row in rows] + [row["chart"] for row in rows])
    for name in os.listdir(output):
        if re.match(r"^ending_[0-9]{3,}\.(wav|svg)$", name) and name not in current:
            os.unlink(os.path.join(output, name))
    return rows


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("input", help="mono 16-bit PCM WAV")
    parser.add_argument("--output", help="output directory; default: INPUT-endings")
    parser.add_argument("--pitch-floor", type=float, default=70.0, metavar="HZ")
    parser.add_argument("--pitch-ceiling", type=float, default=450.0, metavar="HZ")
    parser.add_argument("--min-pause", type=float, default=0.20, metavar="SECONDS")
    parser.add_argument("--min-speech", type=float, default=0.12, metavar="SECONDS")
    parser.add_argument("--force", action="store_true", help="write into a nonempty output directory")
    args = parser.parse_args()
    if not (0 < args.pitch_floor < args.pitch_ceiling):
        parser.error("pitch floor must be positive and below pitch ceiling")
    if args.min_pause <= 0 or args.min_speech <= 0:
        parser.error("pause and speech durations must be positive")
    output = args.output or os.path.splitext(args.input)[0] + "-endings"
    if os.path.isdir(output):
        if os.listdir(output) and not args.force:
            parser.error("output directory is not empty (use --force or another --output)")
    elif os.path.exists(output):
        parser.error("output path exists and is not a directory")
    else:
        os.makedirs(output)
    try:
        rows = analyze(args.input, output, args.pitch_floor, args.pitch_ceiling,
                       args.min_pause, args.min_speech)
    except (IOError, EOFError, ValueError, wave.Error) as error:
        parser.error(str(error) or "not a readable WAV file")
    print("%d candidate endings written to %s" % (len(rows), output))
    for row in rows:
        change = row["terminal_change_st"]
        movement = "%+.2f st" % change if change is not None else "unknown"
        print("%6.2fs  %-9s  %s" % (row["end_s"], row["shape"], movement))


if __name__ == "__main__":
    main()
