// On-TV visualiser: analyse the audio here, send HyperHDR a picture. // // The RTP sink needs a virtual sound device set up on the HyperHDR machine. // This one needs nothing: the TV runs the FFT, renders a small RGB image and // pushes it to HyperHDR's FlatBuffers image input (TCP 19400), exactly as a // video grabber would. HyperHDR maps the image onto the LED layout it already // has, so the lights react to sound with no host-side configuration. // // The trade-off is that HyperHDR's own audio effects are bypassed — the look // is defined here instead. Use the RTP sink when you want HyperHDR's effects, // this one when you want it to just work. #include "sink.h" #include "../common/log.h" #include "../net/hyperion.h" #include #include #include #include #include #define VIZ_MAX_WIDTH 128 #define VIZ_MAX_HEIGHT 128 #define RECONNECT_INTERVAL_SEC 5 typedef enum { VIZ_SPECTRUM, // bars across the width, hue by frequency VIZ_LEVEL, // whole frame lit, colour mixed from band energy VIZ_PULSE, // whole frame lit, brightness follows loudness only } viz_mode_t; typedef struct { hyperion_target_t target; // resolved once at open; reconnects never do DNS char host[128]; int port; int priority; viz_mode_t mode; int width; int height; int fps; float saturation; float floor_level; // minimum brightness so the lights never go fully dark hyperion_client_t* client; time_t last_connect_attempt; char last_error[192]; uint8_t* frame; size_t frame_bytes; struct timespec last_send; dsp_levels_t latest; bool have_levels; unsigned long long frames_sent; unsigned long long connect_failures; } viz_priv_t; // --------------------------------------------------------------------------- // Rendering // --------------------------------------------------------------------------- static void hsv_to_rgb(float h, float s, float v, uint8_t* out) { h = fmodf(h, 1.0f); if (h < 0.0f) h += 1.0f; float i = floorf(h * 6.0f); float f = h * 6.0f - i; float p = v * (1.0f - s); float q = v * (1.0f - f * s); float t = v * (1.0f - (1.0f - f) * s); float r, g, b; switch ((int)i % 6) { case 0: r = v, g = t, b = p; break; case 1: r = q, g = v, b = p; break; case 2: r = p, g = v, b = t; break; case 3: r = p, g = q, b = v; break; case 4: r = t, g = p, b = v; break; default: r = v, g = p, b = q; break; } out[0] = (uint8_t)(r * 255.0f + 0.5f); out[1] = (uint8_t)(g * 255.0f + 0.5f); out[2] = (uint8_t)(b * 255.0f + 0.5f); } static void fill_frame(viz_priv_t* p, const uint8_t rgb[3]) { for (int i = 0; i < p->width * p->height; i++) { p->frame[i * 3 + 0] = rgb[0]; p->frame[i * 3 + 1] = rgb[1]; p->frame[i * 3 + 2] = rgb[2]; } } // Bars rise from the bottom of the image, one group of columns per band, hue // running red (bass) through to violet (treble). static void render_spectrum(viz_priv_t* p, const dsp_levels_t* lv) { memset(p->frame, 0, p->frame_bytes); for (int x = 0; x < p->width; x++) { int band = x * DSP_BANDS / p->width; if (band >= DSP_BANDS) band = DSP_BANDS - 1; float level = lv->bands[band]; if (level < p->floor_level) level = p->floor_level; int lit = (int)(level * (float)p->height + 0.5f); if (lit > p->height) lit = p->height; // 0.0 (red) through 0.8 (violet); avoids wrapping back to red. float hue = 0.8f * ((float)band / (float)(DSP_BANDS - 1)); uint8_t colour[3]; hsv_to_rgb(hue, p->saturation, level, colour); for (int y = 0; y < lit; y++) { int row = p->height - 1 - y; // row 0 is the top of the image uint8_t* px = &p->frame[((size_t)row * p->width + x) * 3]; px[0] = colour[0]; px[1] = colour[1]; px[2] = colour[2]; } } } // Splits the spectrum into three groups and treats them as an RGB mix, which // gives bass-heavy content a warm cast and bright content a cool one. static void render_level(viz_priv_t* p, const dsp_levels_t* lv) { float low = 0.0f, mid = 0.0f, high = 0.0f; const int third = DSP_BANDS / 3; for (int b = 0; b < DSP_BANDS; b++) { if (b < third) low += lv->bands[b]; else if (b < third * 2) mid += lv->bands[b]; else high += lv->bands[b]; } low /= (float)third; mid /= (float)third; high /= (float)(DSP_BANDS - third * 2); float strongest = low > mid ? low : mid; if (high > strongest) strongest = high; if (strongest < 0.001f) strongest = 0.001f; float brightness = lv->rms * 3.0f; // RMS of music rarely exceeds ~0.33 if (brightness > 1.0f) brightness = 1.0f; if (brightness < p->floor_level) brightness = p->floor_level; uint8_t rgb[3] = { (uint8_t)(low / strongest * brightness * 255.0f), (uint8_t)(mid / strongest * brightness * 255.0f), (uint8_t)(high / strongest * brightness * 255.0f), }; fill_frame(p, rgb); } static void render_pulse(viz_priv_t* p, const dsp_levels_t* lv) { float brightness = lv->peak; if (brightness < p->floor_level) brightness = p->floor_level; uint8_t rgb[3]; // Warm white that shifts slightly warmer as it gets quieter. hsv_to_rgb(0.09f, p->saturation * 0.5f, brightness, rgb); fill_frame(p, rgb); } // --------------------------------------------------------------------------- // Never blocks: the connect is started here and completed by hyperion_pump() // on later blocks, because this runs on the capture thread. static bool ensure_connected(viz_priv_t* p) { if (p->client) return true; time_t now = time(NULL); if (now - p->last_connect_attempt < RECONNECT_INTERVAL_SEC) return false; p->last_connect_attempt = now; char err[192] = { 0 }; p->client = hyperion_connect(&p->target, "lgtv-audio-cap", p->priority, err, sizeof(err)); if (!p->client) { p->connect_failures++; // Only log when the message changes, so an unreachable host does not // spam one line every five seconds forever. if (strcmp(err, p->last_error) != 0) { WARN("HyperHDR visualiser: %s", err); snprintf(p->last_error, sizeof(p->last_error), "%s", err); } return false; } p->last_error[0] = '\0'; return true; } static bool frame_due(viz_priv_t* p) { struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); double elapsed = (double)(now.tv_sec - p->last_send.tv_sec) + (double)(now.tv_nsec - p->last_send.tv_nsec) / 1e9; if (elapsed < 1.0 / (double)p->fps) return false; p->last_send = now; return true; } static void viz_write(sink_t* s, const int16_t* pcm, int frames, const dsp_levels_t* levels) { (void)pcm; (void)frames; viz_priv_t* p = s->priv; if (levels) { p->latest = *levels; p->have_levels = true; } if (!ensure_connected(p)) return; if (!hyperion_pump(p->client)) { const char* why = hyperion_last_error(p->client); WARN("HyperHDR visualiser disconnected: %s", why ? why : "unknown"); snprintf(p->last_error, sizeof(p->last_error), "%s", why ? why : "disconnected"); hyperion_disconnect(p->client); p->client = NULL; return; } // Capture blocks arrive far faster than the LEDs need updating; rate-limit // so we are not shipping an image every 10 ms over the network. if (!p->have_levels || !frame_due(p)) return; switch (p->mode) { case VIZ_SPECTRUM: render_spectrum(p, &p->latest); break; case VIZ_LEVEL: render_level(p, &p->latest); break; case VIZ_PULSE: render_pulse(p, &p->latest); break; } if (!hyperion_send_image(p->client, p->frame, p->width, p->height)) { const char* why = hyperion_last_error(p->client); WARN("HyperHDR visualiser send failed: %s", why ? why : "unknown"); hyperion_disconnect(p->client); p->client = NULL; return; } if (hyperion_registered(p->client)) p->frames_sent++; } static const char* mode_name(viz_mode_t m) { switch (m) { case VIZ_SPECTRUM: return "spectrum"; case VIZ_LEVEL: return "level"; default: return "pulse"; } } static void viz_status(sink_t* s, json_writer_t* w) { viz_priv_t* p = s->priv; jw_str(w, "target", p->host); jw_int(w, "port", p->port); jw_int(w, "priority", p->priority); jw_str(w, "mode", mode_name(p->mode)); jw_int(w, "width", p->width); jw_int(w, "height", p->height); jw_int(w, "fps", p->fps); jw_bool(w, "connected", hyperion_connected(p->client)); jw_bool(w, "registered", p->client && hyperion_registered(p->client)); jw_int(w, "framesSent", (long long)p->frames_sent); jw_int(w, "connectFailures", (long long)p->connect_failures); if (p->last_error[0]) jw_str(w, "lastError", p->last_error); else jw_null(w, "lastError"); } static void viz_close(sink_t* s) { viz_priv_t* p = s->priv; if (p) { if (p->client) hyperion_disconnect(p->client); free(p->frame); free(p); } free(s); } static int clamp_int(int v, int lo, int hi) { return v < lo ? lo : (v > hi ? hi : v); } static sink_t* viz_open(const json_value_t* cfg, const audio_format_t* fmt, char* err, size_t errlen) { const json_value_t* sc = json_get(cfg, "hyperhdrViz"); const char* host = json_str(sc, "host", NULL); // Fall back to the audio sink's host so the common case needs one address. if (!host || !*host) host = json_str(json_get(cfg, "hyperhdr"), "host", NULL); if (!host || !*host) { snprintf(err, errlen, "set the HyperHDR host address first"); return NULL; } viz_priv_t* p = calloc(1, sizeof(*p)); sink_t* s = calloc(1, sizeof(*s)); if (!p || !s) { free(p); free(s); snprintf(err, errlen, "out of memory"); return NULL; } snprintf(p->host, sizeof(p->host), "%s", host); p->port = clamp_int(json_int(sc, "port", 19400), 1, 65535); if (!hyperion_resolve(p->host, p->port, &p->target, err, errlen)) { free(p); free(s); return NULL; } p->priority = clamp_int(json_int(sc, "priority", 150), 1, 253); p->width = clamp_int(json_int(sc, "width", 64), 4, VIZ_MAX_WIDTH); p->height = clamp_int(json_int(sc, "height", 36), 4, VIZ_MAX_HEIGHT); p->fps = clamp_int(json_int(sc, "fps", 30), 1, 60); p->saturation = (float)json_num(sc, "saturation", 1.0); p->floor_level = (float)json_num(sc, "minBrightness", 0.02); const char* mode = json_str(sc, "mode", "spectrum"); if (strcmp(mode, "level") == 0) p->mode = VIZ_LEVEL; else if (strcmp(mode, "pulse") == 0) p->mode = VIZ_PULSE; else p->mode = VIZ_SPECTRUM; p->frame_bytes = (size_t)p->width * (size_t)p->height * 3; p->frame = calloc(1, p->frame_bytes); if (!p->frame) { snprintf(err, errlen, "out of memory allocating %dx%d frame", p->width, p->height); free(p); free(s); return NULL; } clock_gettime(CLOCK_MONOTONIC, &p->last_send); s->driver = &sink_driver_hyperhdr_viz; s->priv = p; s->fmt = *fmt; s->write = viz_write; s->status = viz_status; s->close = viz_close; INFO("HyperHDR visualiser sink: %s:%d mode=%s %dx%d @%d fps priority=%d", p->host, p->port, mode_name(p->mode), p->width, p->height, p->fps, p->priority); return s; } const sink_driver_t sink_driver_hyperhdr_viz = { .id = "hyperhdrViz", .name = "HyperHDR visualiser (FlatBuffers)", .description = "Runs the spectrum analysis on the TV and pushes images to HyperHDR. No host setup.", .open = viz_open, };