lgtv_audio_cap/native/src/sinks/sink_hyperhdr_viz.c

// On-TV visualiser: analyse the audio here, send HyperHDR a picture.
//
// The RTP sink needs a virtual sound device set up on the HyperHDR machine.
// This one needs nothing: the TV runs the FFT, renders a small RGB image and
// pushes it to HyperHDR's FlatBuffers image input (TCP 19400), exactly as a
// video grabber would. HyperHDR maps the image onto the LED layout it already
// has, so the lights react to sound with no host-side configuration.
//
// The trade-off is that HyperHDR's own audio effects are bypassed — the look
// is defined here instead. Use the RTP sink when you want HyperHDR's effects,
// this one when you want it to just work.

#include "sink.h"
#include "../common/log.h"
#include "../net/hyperion.h"

#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

#define VIZ_MAX_WIDTH 128
#define VIZ_MAX_HEIGHT 128
#define RECONNECT_INTERVAL_SEC 5

typedef enum {
    VIZ_SPECTRUM, // bars across the width, hue by frequency
    VIZ_LEVEL, // whole frame lit, colour mixed from band energy
    VIZ_PULSE, // whole frame lit, brightness follows loudness only
} viz_mode_t;

typedef struct {
    hyperion_target_t target; // resolved once at open; reconnects never do DNS
    char host[128];
    int port;
    int priority;
    viz_mode_t mode;

    int width;
    int height;
    int fps;
    float saturation;
    float floor_level; // minimum brightness so the lights never go fully dark

    hyperion_client_t* client;
    time_t last_connect_attempt;
    char last_error[192];

    uint8_t* frame;
    size_t frame_bytes;

    struct timespec last_send;
    dsp_levels_t latest;
    bool have_levels;

    unsigned long long frames_sent;
    unsigned long long connect_failures;
} viz_priv_t;

// ---------------------------------------------------------------------------
// Rendering
// ---------------------------------------------------------------------------

static void hsv_to_rgb(float h, float s, float v, uint8_t* out)
{
    h = fmodf(h, 1.0f);
    if (h < 0.0f)
        h += 1.0f;

    float i = floorf(h * 6.0f);
    float f = h * 6.0f - i;
    float p = v * (1.0f - s);
    float q = v * (1.0f - f * s);
    float t = v * (1.0f - (1.0f - f) * s);

    float r, g, b;
    switch ((int)i % 6) {
    case 0:
        r = v, g = t, b = p;
        break;
    case 1:
        r = q, g = v, b = p;
        break;
    case 2:
        r = p, g = v, b = t;
        break;
    case 3:
        r = p, g = q, b = v;
        break;
    case 4:
        r = t, g = p, b = v;
        break;
    default:
        r = v, g = p, b = q;
        break;
    }

    out[0] = (uint8_t)(r * 255.0f + 0.5f);
    out[1] = (uint8_t)(g * 255.0f + 0.5f);
    out[2] = (uint8_t)(b * 255.0f + 0.5f);
}

static void fill_frame(viz_priv_t* p, const uint8_t rgb[3])
{
    for (int i = 0; i < p->width * p->height; i++) {
        p->frame[i * 3 + 0] = rgb[0];
        p->frame[i * 3 + 1] = rgb[1];
        p->frame[i * 3 + 2] = rgb[2];
    }
}

// Bars rise from the bottom of the image, one group of columns per band, hue
// running red (bass) through to violet (treble).
static void render_spectrum(viz_priv_t* p, const dsp_levels_t* lv)
{
    memset(p->frame, 0, p->frame_bytes);

    for (int x = 0; x < p->width; x++) {
        int band = x * DSP_BANDS / p->width;
        if (band >= DSP_BANDS)
            band = DSP_BANDS - 1;

        float level = lv->bands[band];
        if (level < p->floor_level)
            level = p->floor_level;

        int lit = (int)(level * (float)p->height + 0.5f);
        if (lit > p->height)
            lit = p->height;

        // 0.0 (red) through 0.8 (violet); avoids wrapping back to red.
        float hue = 0.8f * ((float)band / (float)(DSP_BANDS - 1));
        uint8_t colour[3];
        hsv_to_rgb(hue, p->saturation, level, colour);

        for (int y = 0; y < lit; y++) {
            int row = p->height - 1 - y; // row 0 is the top of the image
            uint8_t* px = &p->frame[((size_t)row * p->width + x) * 3];
            px[0] = colour[0];
            px[1] = colour[1];
            px[2] = colour[2];
        }
    }
}

// Splits the spectrum into three groups and treats them as an RGB mix, which
// gives bass-heavy content a warm cast and bright content a cool one.
static void render_level(viz_priv_t* p, const dsp_levels_t* lv)
{
    float low = 0.0f, mid = 0.0f, high = 0.0f;
    const int third = DSP_BANDS / 3;

    for (int b = 0; b < DSP_BANDS; b++) {
        if (b < third)
            low += lv->bands[b];
        else if (b < third * 2)
            mid += lv->bands[b];
        else
            high += lv->bands[b];
    }
    low /= (float)third;
    mid /= (float)third;
    high /= (float)(DSP_BANDS - third * 2);

    float strongest = low > mid ? low : mid;
    if (high > strongest)
        strongest = high;
    if (strongest < 0.001f)
        strongest = 0.001f;

    float brightness = lv->rms * 3.0f; // RMS of music rarely exceeds ~0.33
    if (brightness > 1.0f)
        brightness = 1.0f;
    if (brightness < p->floor_level)
        brightness = p->floor_level;

    uint8_t rgb[3] = {
        (uint8_t)(low / strongest * brightness * 255.0f),
        (uint8_t)(mid / strongest * brightness * 255.0f),
        (uint8_t)(high / strongest * brightness * 255.0f),
    };
    fill_frame(p, rgb);
}

static void render_pulse(viz_priv_t* p, const dsp_levels_t* lv)
{
    float brightness = lv->peak;
    if (brightness < p->floor_level)
        brightness = p->floor_level;

    uint8_t rgb[3];
    // Warm white that shifts slightly warmer as it gets quieter.
    hsv_to_rgb(0.09f, p->saturation * 0.5f, brightness, rgb);
    fill_frame(p, rgb);
}

// ---------------------------------------------------------------------------

// Never blocks: the connect is started here and completed by hyperion_pump()
// on later blocks, because this runs on the capture thread.
static bool ensure_connected(viz_priv_t* p)
{
    if (p->client)
        return true;

    time_t now = time(NULL);
    if (now - p->last_connect_attempt < RECONNECT_INTERVAL_SEC)
        return false;
    p->last_connect_attempt = now;

    char err[192] = { 0 };
    p->client = hyperion_connect(&p->target, "lgtv-audio-cap", p->priority, err, sizeof(err));
    if (!p->client) {
        p->connect_failures++;
        // Only log when the message changes, so an unreachable host does not
        // spam one line every five seconds forever.
        if (strcmp(err, p->last_error) != 0) {
            WARN("HyperHDR visualiser: %s", err);
            snprintf(p->last_error, sizeof(p->last_error), "%s", err);
        }
        return false;
    }

    p->last_error[0] = '\0';
    return true;
}

static bool frame_due(viz_priv_t* p)
{
    struct timespec now;
    clock_gettime(CLOCK_MONOTONIC, &now);

    double elapsed = (double)(now.tv_sec - p->last_send.tv_sec)
        + (double)(now.tv_nsec - p->last_send.tv_nsec) / 1e9;
    if (elapsed < 1.0 / (double)p->fps)
        return false;

    p->last_send = now;
    return true;
}

static void viz_write(sink_t* s, const int16_t* pcm, int frames, const dsp_levels_t* levels)
{
    (void)pcm;
    (void)frames;
    viz_priv_t* p = s->priv;

    if (levels) {
        p->latest = *levels;
        p->have_levels = true;
    }

    if (!ensure_connected(p))
        return;

    if (!hyperion_pump(p->client)) {
        const char* why = hyperion_last_error(p->client);
        WARN("HyperHDR visualiser disconnected: %s", why ? why : "unknown");
        snprintf(p->last_error, sizeof(p->last_error), "%s", why ? why : "disconnected");
        hyperion_disconnect(p->client);
        p->client = NULL;
        return;
    }

    // Capture blocks arrive far faster than the LEDs need updating; rate-limit
    // so we are not shipping an image every 10 ms over the network.
    if (!p->have_levels || !frame_due(p))
        return;

    switch (p->mode) {
    case VIZ_SPECTRUM:
        render_spectrum(p, &p->latest);
        break;
    case VIZ_LEVEL:
        render_level(p, &p->latest);
        break;
    case VIZ_PULSE:
        render_pulse(p, &p->latest);
        break;
    }

    if (!hyperion_send_image(p->client, p->frame, p->width, p->height)) {
        const char* why = hyperion_last_error(p->client);
        WARN("HyperHDR visualiser send failed: %s", why ? why : "unknown");
        hyperion_disconnect(p->client);
        p->client = NULL;
        return;
    }

    if (hyperion_registered(p->client))
        p->frames_sent++;
}

static const char* mode_name(viz_mode_t m)
{
    switch (m) {
    case VIZ_SPECTRUM:
        return "spectrum";
    case VIZ_LEVEL:
        return "level";
    default:
        return "pulse";
    }
}

static void viz_status(sink_t* s, json_writer_t* w)
{
    viz_priv_t* p = s->priv;
    jw_str(w, "target", p->host);
    jw_int(w, "port", p->port);
    jw_int(w, "priority", p->priority);
    jw_str(w, "mode", mode_name(p->mode));
    jw_int(w, "width", p->width);
    jw_int(w, "height", p->height);
    jw_int(w, "fps", p->fps);
    jw_bool(w, "connected", hyperion_connected(p->client));
    jw_bool(w, "registered", p->client && hyperion_registered(p->client));
    jw_int(w, "framesSent", (long long)p->frames_sent);
    jw_int(w, "connectFailures", (long long)p->connect_failures);
    if (p->last_error[0])
        jw_str(w, "lastError", p->last_error);
    else
        jw_null(w, "lastError");
}

static void viz_close(sink_t* s)
{
    viz_priv_t* p = s->priv;
    if (p) {
        if (p->client)
            hyperion_disconnect(p->client);
        free(p->frame);
        free(p);
    }
    free(s);
}

static int clamp_int(int v, int lo, int hi)
{
    return v < lo ? lo : (v > hi ? hi : v);
}

static sink_t* viz_open(const json_value_t* cfg, const audio_format_t* fmt, char* err, size_t errlen)
{
    const json_value_t* sc = json_get(cfg, "hyperhdrViz");
    const char* host = json_str(sc, "host", NULL);

    // Fall back to the audio sink's host so the common case needs one address.
    if (!host || !*host)
        host = json_str(json_get(cfg, "hyperhdr"), "host", NULL);
    if (!host || !*host) {
        snprintf(err, errlen, "set the HyperHDR host address first");
        return NULL;
    }

    viz_priv_t* p = calloc(1, sizeof(*p));
    sink_t* s = calloc(1, sizeof(*s));
    if (!p || !s) {
        free(p);
        free(s);
        snprintf(err, errlen, "out of memory");
        return NULL;
    }

    snprintf(p->host, sizeof(p->host), "%s", host);
    p->port = clamp_int(json_int(sc, "port", 19400), 1, 65535);
    if (!hyperion_resolve(p->host, p->port, &p->target, err, errlen)) {
        free(p);
        free(s);
        return NULL;
    }
    p->priority = clamp_int(json_int(sc, "priority", 150), 1, 253);
    p->width = clamp_int(json_int(sc, "width", 64), 4, VIZ_MAX_WIDTH);
    p->height = clamp_int(json_int(sc, "height", 36), 4, VIZ_MAX_HEIGHT);
    p->fps = clamp_int(json_int(sc, "fps", 30), 1, 60);
    p->saturation = (float)json_num(sc, "saturation", 1.0);
    p->floor_level = (float)json_num(sc, "minBrightness", 0.02);

    const char* mode = json_str(sc, "mode", "spectrum");
    if (strcmp(mode, "level") == 0)
        p->mode = VIZ_LEVEL;
    else if (strcmp(mode, "pulse") == 0)
        p->mode = VIZ_PULSE;
    else
        p->mode = VIZ_SPECTRUM;

    p->frame_bytes = (size_t)p->width * (size_t)p->height * 3;
    p->frame = calloc(1, p->frame_bytes);
    if (!p->frame) {
        snprintf(err, errlen, "out of memory allocating %dx%d frame", p->width, p->height);
        free(p);
        free(s);
        return NULL;
    }

    clock_gettime(CLOCK_MONOTONIC, &p->last_send);

    s->driver = &sink_driver_hyperhdr_viz;
    s->priv = p;
    s->fmt = *fmt;
    s->write = viz_write;
    s->status = viz_status;
    s->close = viz_close;

    INFO("HyperHDR visualiser sink: %s:%d mode=%s %dx%d @%d fps priority=%d", p->host,
        p->port, mode_name(p->mode), p->width, p->height, p->fps, p->priority);
    return s;
}

const sink_driver_t sink_driver_hyperhdr_viz = {
    .id = "hyperhdrViz",
    .name = "HyperHDR visualiser (FlatBuffers)",
    .description = "Runs the spectrum analysis on the TV and pushes images to HyperHDR. No host setup.",
    .open = viz_open,
};