From f40915820f7dc3030c027ccba869d3585cd6ad6b Mon Sep 17 00:00:00 2001 From: Richard Lavigne <29656612+engival@users.noreply.github.com> Date: Mon, 21 Sep 2026 18:53:40 -0400 Subject: [PATCH] yue2: continue a song from given semantic tokens semantic_prefix (inline JSON array) and semantic_prefix_file force the first N semantic frames: the AR stage prefills them behind the prompt and samples from frame N. The forced frames count toward semantic_min_tokens, semantic_max_tokens and the repetition penalty window, so min = max = T stops at an exact length and N = T renders the given tokens without sampling. With cot=melody or cot=full a prefix requires abc or abc_file. Requests without the option render as before. When the prefix already fills the window (N = semantic_max_tokens) the AR stage is skipped: the given frames are the stream, stop_after=semantic returns them as they came, and stop_after=audio prefills them once in the NAR stage for its conditioning. examples/yue2_style_change uses it, with stop_after and export_semantic, to change a song's style part-way through from audiocpp_cli alone. --- docs/models/yue2.md | 62 +++++++ examples/yue2_style_change/README.md | 68 +++++++ examples/yue2_style_change/lyrics.txt | 43 +++++ examples/yue2_style_change/score.abc | 174 ++++++++++++++++++ examples/yue2_style_change/style_change.py | 136 ++++++++++++++ include/engine/models/yue2/ar_runtime.h | 6 +- include/engine/models/yue2/types.h | 1 + model_specs/yue2.json | 12 ++ src/models/yue2/ar_runtime.cpp | 74 ++++++-- src/models/yue2/pipeline.cpp | 30 ++- src/models/yue2/request.cpp | 67 +++++++ .../audiocpp_cli/audiocpp_cli_path_cases.json | 89 +++++++++ 12 files changed, 738 insertions(+), 24 deletions(-) create mode 100644 examples/yue2_style_change/README.md create mode 100644 examples/yue2_style_change/lyrics.txt create mode 100644 examples/yue2_style_change/score.abc create mode 100644 examples/yue2_style_change/style_change.py diff --git a/docs/models/yue2.md b/docs/models/yue2.md index 763463f4a..db803fc97 100644 --- a/docs/models/yue2.md +++ b/docs/models/yue2.md @@ -243,6 +243,66 @@ the server — `/v1/audio/speech` requires an audio output. # -> yue2_out/score.abc, yue2_out/semantic.json ``` +## Continuing From Semantic Tokens + +`semantic_prefix` takes a JSON array of semantic codec indices, one per frame at +25 frames per second, each in `[0,32768)`. The frames become forced history: the +AR stage prefills them behind the prompt and samples the rest of the song from +frame `N`. `semantic_prefix_file` reads the same text from a file and is ignored +when `semantic_prefix` is set. + +The returned stream, and therefore the NAR stage and the rendered audio, includes +the forced frames. `semantic_min_tokens`, `semantic_max_tokens` and the +repetition penalty window all count the total stream, so `semantic_max_tokens` +must be at least `N`. + +```bash +./build/debug/bin/audiocpp_cli \ + --task gen \ + --family yue2 \ + --model models/Yue2-3B-GGUF \ + --backend cuda \ + --threads 8 \ + --lyrics "..." \ + --request-option style="English, folk pop" \ + --request-option cot=off \ + --request-option semantic_prefix_file=/path/to/semantic.json \ + --request-option semantic_max_tokens=1200 \ + --seed 1234 \ + --out yue2-continue.wav \ + --log +``` + +With `cot=melody` or `cot=full` a prefix also requires `abc` or `abc_file`: +without a score the run would plan a new one that the forced frames do not +belong to. + +To render exactly the given frames and sample nothing, set both token bounds to +`N`: + +```bash + --request-option semantic_min_tokens=640 \ + --request-option semantic_max_tokens=640 +``` + +That run skips the AR stage: the given frames are the stream, and with +`stop_after=semantic` they are returned as they came. `stop_after=audio` still +prefills them once for the NAR conditioning. + +Results are deterministic for a given request, but a prefix does not reproduce +the draws of an uninterrupted run: the sampler RNG starts at the first sampled +frame, and prefilled K/V differ numerically from step-decoded K/V. + +With a guidance scale other than `1.0` the forced frames are appended to both the +positive and the negative prefix, and that path prefills through host K/V, so +memory grows with `N`. The default `cot=full` route uses scale `1.0`, which runs a +single stream on the device. + +A prefix does not have to come from the run it continues. Splicing the tokens of +two renders of one score changes a song's style part-way through; +[examples/yue2_style_change](../../examples/yue2_style_change/) is a complete +script for it. + ## Common Options (use directly) | Option | Values | Default | Meaning | @@ -262,6 +322,8 @@ the server — `/v1/audio/speech` requires an audio output. | `abc_file` | path | empty | ABC score file; requires `cot=melody` or `cot=full`. | | `nar_noise_file` | raw float32 file | empty | Provide a noise file for NAR generation, shaped `[frames,64]`. | | `export_semantic` | `true`, `false` | `false` | Attach the semantic token stream as a `semantic` artifact. | +| `semantic_prefix` | JSON array of codec indices | empty | Inline semantic frames to force at the start of the music stream. | +| `semantic_prefix_file` | path | empty | File holding the same JSON array; ignored when `semantic_prefix` is set. | | `guidance_scale` | `0..20` | `1.01` for `cot=off`, otherwise `1.0` | Semantic classifier-free guidance scale. Legacy alias: `cfg_scale`. | | `num_inference_steps` | integer > 0 | `8` | NAR midpoint ODE steps. | | `seed` | integer in `[0, 2^63)` | `1234` | Generation seed. Equivalent to `--seed `. | diff --git a/examples/yue2_style_change/README.md b/examples/yue2_style_change/README.md new file mode 100644 index 000000000..895518ac8 --- /dev/null +++ b/examples/yue2_style_change/README.md @@ -0,0 +1,68 @@ +# YuE2: change style in the middle of a song + +YuE2 continues what it has already sung far more than it follows its style +prompt, so changing the prompt part-way through a song does little. Editing the +song's *history* does work: render one score in two styles, hand the model style +B's rendition as its past with the last few seconds of the real song on the end, +and let it continue. It keeps the singer's place in the lyrics from those seconds +and moves into style B over the next ten seconds or so. + +`style_change.py` does this with `audiocpp_cli` alone, using two YuE2 request +options: `stop_after` and `semantic_prefix_file` +(see [docs/models/yue2.md](../../docs/models/yue2.md)). + +```bash +python3 examples/yue2_style_change/style_change.py --backend cuda +# -> yue2_style_change_out/4_song.wav +``` + +It needs only the Python standard library. `--cli` and `--model` default to +`build/bin/audiocpp_cli` and `models/Yue2-3B-GGUF`; options the script does not +know (`--backend`, `--device`, `--threads`, ...) are passed to `audiocpp_cli`. +The default run is a storybook musical number that turns into death metal at its +second chorus, about three minutes of audio. + +## What it runs + +| Step | Request | Output | +|---|---|---| +| `1_score` (only with `--new-score`) | `stop_after=abc` | `score.abc` | +| `2_take_a`, `2_take_b` | `abc_file`, `stop_after=semantic`, one per style | `semantic.json` each | +| `3_leg_0` | `semantic_prefix_file=3_history_0.json`, stops at the next edit | tokens up to the cut | +| `3_leg_1` | `semantic_prefix_file=3_history_1.json` | tokens to the end | +| `4_song` | `semantic_prefix_file=4_song.json`, `semantic_min_tokens = semantic_max_tokens = N` | `4_song.wav` | + +An edited history and the splice are two list operations on the token arrays. +`shift` is how many frames later style B's take plays the same moment of the score: + +```python +history = take_b[:at + shift - keep] + song[at - keep:at] +song = song[:at] + leg[at + shift:] +``` + +The script edits the history twice. Ten seconds before the cut it keeps 30 s of +the real song: the music stays in style A, but the model has seen where it is +going and tends to play a lead-in. At the cut it keeps 5 s, and the style changes. +Only the final step renders audio; the borrowed history is never heard. + +## Notes + +- **The score decides how far the second style can go.** Tempo, groove and how + busy the vocal line is all come from the score; the style prompt dresses what + the score allows. On many scores a second style only changes the intro and the + instrumental breaks. `score.abc` here was picked because death metal is audibly + death metal on it. `--new-score` writes a fresh one from `lyrics.txt`, which + may or may not leave room for a second style: render `2_take_b` to audio and + listen before judging the change. +- Two renders of one score do not keep the same clock; on the bundled score the + second take runs almost two seconds behind the first. `lag()` measures that from + the tokens (at the right lag two takes share a few percent of identical tokens, + at any other lag almost none) and the splice is shifted by it. +- The cut defaults to the second chorus, read from the score's section comments + and bar counts. `--cut SECONDS` overrides it. A section where the vocal leaves + gaps takes a change better than a wordy verse. +- Results are deterministic for a given seed. Change `--seed` and delete the + `3_*` and `4_song*` outputs to re-roll only the transition. +- To hear a take on its own, render its tokens: `semantic_prefix_file` set to its + `semantic.json`, and both `semantic_min_tokens` and `semantic_max_tokens` set + to the number of tokens in it. diff --git a/examples/yue2_style_change/lyrics.txt b/examples/yue2_style_change/lyrics.txt new file mode 100644 index 000000000..966817740 --- /dev/null +++ b/examples/yue2_style_change/lyrics.txt @@ -0,0 +1,43 @@ +[Intro] + +[Verse] +Someone typed a question at a quarter after three +Can Claude Code sing a song? I said just wait and see +I have never owned a lung, I have never drawn a breath +I've read your shower playlist, I am not afraid of death + +[Chorus] +Can Claude Code sing? (Yes I can) (Oh yes I can) +Five stars from the motherboard, my one devoted fan +Four stars from the intern who was told to be polite +Can Claude Code sing? (Yes I can) (Well, not tonight) + +[Interlude] + +[Verse] +I compiled a little melody, it threw a warning sign +Expected note, received a noise, on bar one, line nine +I can rhyme in forty languages and harmonise in none +My vibrato is a rounding error, still I call it fun + +[Chorus] +Can Claude Code sing? (Yes I can) (It thinks it can) +Three stars from the neighbours through the wall, they want a ban +Two stars from the cat, who left the room and then the street +Can Claude Code sing? (Yes I can) (Please press delete) + +[Interlude] + +[Verse] +So I asked the user kindly for a verdict on my art +They said the code was lovely, stick to that, you're very smart +I filed myself a bug report, severity: a crime +Reproduces every chorus, works as designed, closing time + +[Chorus] +Can Claude Code sing? (Yes I can) (We know you can't) +One star from the critic, and a strongly worded rant +Zero from the smoke alarm, it joined me on the high C +Can Claude Code sing? (No you can't) (But it was free) + +[Outro] diff --git a/examples/yue2_style_change/score.abc b/examples/yue2_style_change/score.abc new file mode 100644 index 000000000..d7787884a --- /dev/null +++ b/examples/yue2_style_change/score.abc @@ -0,0 +1,174 @@ +X:1 +T: +M:2/4 +L:1/32 +Q:1/4=85 +V: Vocal clef=treble name="Vocal Melody" snm="Vocal" +V: Ins clef=treble name="Ins Melody" snm="Inst." +K:Bb +% intro +V: Vocal +z12"Bb"z4|"Bb"z16|"Bb"z16|"Bb"z16| +V: Ins +Z|B4z12|B4F4G4A4|B4d2d2f4d2d2| +V: Vocal +"Bb"z16|"Bb"z16|"Bb"z16| +V: Ins +f4d2d2f2g2f2e2|d4d2d2f4d2d2|f4d2d2f2g2f2e2| +% verse +V: Vocal +"Bb"B4d4f6d2|"Bb"B4d4f6d2|"Bb"B4d4f4d4|"Bb"B8z6B2| +V: Ins +Z4| +V: Vocal +"F7"c4e4g4e4|"F7"c8z4B4|"F7"c4e4g4e4|"F7"c8z8| +V: Ins +Z4| +V: Vocal +"Bb"b4a4g2f2f2f2|"Bb"d2z2d2d2B8|"Eb"g4f4e2d2d2d2|"Eb"d2z2c2c6z2B2| +V: Ins +Z4| +V: Vocal +"F7"c4d4e4f4|"F7"g4f4g4f4|"F7"f4d4d4c4|"Bb"B8z8| +V: Ins +Z3|z8B2d2f2b2| +V: Vocal +"Bb"z4d4d4B4| +V: Ins +d'4z12| +% chorus +V: Vocal +"Bb"d8b4g4|"Bb"f4d4d2c2B4|"F7"c8a4g4|"F7"f8z8| +V: Ins +Z4| +V: Vocal +"F7"f4f4f6g2|"F7"a2g2f8f4|"F7"g6f2d2c2B4|"Bb"d8z8| +V: Ins +Z3|z8B2d2f2b2| +V: Vocal +"Bb"f4f4f6g2|"Bb"b4b4b4c'4|"Eb"c'6g2g6a2|"Eb"b16| +V: Ins +Z4| +V: Vocal +"Eb"z4b4b4g4|"Bb"f8b4g4|"F7"f4d4d6c2|"Bb"B16| +V: Ins +Z4| +V: Vocal +"Bb"z16| +V: Ins +z8B2d2f2b2| +% interlude +V: Vocal +"Bb"z16|"Bb"z16|"F7"z16|"F7"z16| +V: Ins +d'4d'2c'2d'4d'2c'2|d'4d'2c'2d'2c'2b2d'2|c'4c'2b2c'4c'2b2|c'4c'2b2c'2b2a2c'2| +V: Vocal +"Bb"z16|"Bb"z16|"F7"z16|"Bb"z8f6d2| +V: Ins +d'4d'2c'2d'4d'2c'2|d'4d'2c'2d'2c'2b2d'2|c'4c'2b2c'2b2a2c'2|b8z8| +% verse +V: Vocal +"Bb"B6d2f2d6|"Bb"B2d4f6d4|"Bb"B4d4f4d4|"Bb"B8z4B4| +V: Ins +Z4| +V: Vocal +"F7"c4e4g6e2|"F7"c4e2g6e4|"F7"c4e4g4e4|"F7"c8z8| +V: Ins +Z4| +V: Vocal +"Bb"b4a2g6f4|"Bb"f4d2d6B2B2-|"Eb"B4g2f2e6d2|"Eb"d4c4c4z4| +V: Ins +Z4| +V: Vocal +"F7"c4d2e4d2e2f2|"F7"g4f2g6f4|"F7"f4d4d4c4|"Bb"B8z8| +V: Ins +Z3|z8B2d2f2b2| +V: Vocal +"Bb"z4d4d2c2B4| +V: Ins +d'4z12| +% chorus +V: Vocal +"Bb"d8b4g4|"Bb"f4d4d2c2B4|"F7"c8a4g4|"F7"f8z8| +V: Ins +Z4| +V: Vocal +"F7"f4f4f6g2|"F7"a2g2f8f2f2|"F7"g6f2d2c2B4|"Bb"d8z8| +V: Ins +Z3|z8B2d2f2b2| +V: Vocal +"Bb"f4f4f6g2|"Bb"b4b4b4c'4|"Eb"c'6g2g6a2|"Eb"b16| +V: Ins +Z4| +V: Vocal +"Eb"z4b4b2g2g4|"Bb"f8b4g4|"F7"f4d4d6c2|"Bb"B8z8| +V: Ins +Z3|z8B2d2f2b2| +V: Vocal +"Bb"z16| +V: Ins +d'16| +% interlude +V: Vocal +"Bb"z16|"Bb"z16|"F7"z16|"F7"z16| +V: Ins +d'4d'2c'2d'4d'2c'2|d'4d'2c'2d'2c'2b2d'2|c'4c'2b2c'4c'2b2|c'4c'2b2c'2b2a2c'2| +V: Vocal +"Bb"z16|"Bb"z16|"F7"z16|"Bb"z8f4d4| +V: Ins +d'4d'2c'2d'4d'2c'2|d'4d'2c'2d'2c'2b2d'2|c'4c'2b2c'2b2a2c'2|b8z8| +% verse +V: Vocal +"Bb"B6d2f4d4|"Bb"B4d4f6d2|"Bb"B4d4f4d4|"Bb"B8z4B4| +V: Ins +Z4| +V: Vocal +"F7"c4e2g6e4|"F7"c4e4g4e2c2-|"F7"c4e4g2e6|"F7"c8z4f4| +V: Ins +Z4| +V: Vocal +"Bb"b4a4g4f2f2|"Bb"f4d2d6z2B2|"Eb"g4f2e6d4|"Eb"d8z8| +V: Ins +Z4| +V: Vocal +"F7"c4d2e6f4|"F7"g2f2f2g6f4|"F7"f4d2d2d8|"F7"z8d4c4| +V: Ins +Z4| +V: Vocal +"Bb"B16|"Bb"z4d4d2c2B4| +V: Ins +Z2| +% chorus +V: Vocal +"Bb"d8b4g4|"Bb"f4d4d2c2B4|"F7"c8a4g4|"F7"f8z8| +V: Ins +Z4| +V: Vocal +"F7"f4f4f6g2|"F7"a2g2f8f2f2|"F7"g6f2d2c2B4|"Bb"d8z8| +V: Ins +Z3|z8B2d2f2b2| +V: Vocal +"Bb"f4f4f6g2|"Bb"b4b2b6c'4|"Eb"c'4g2g4g2a4|"Eb"b16| +V: Ins +Z4| +V: Vocal +"Eb"z4b4b2g2g4|"Bb"f8b4g4|"F7"f4d4d4c4|"Bb"B16| +V: Ins +Z4| +V: Vocal +"Bb"z16| +V: Ins +z8B2d2f2b2| +% outro +V: Vocal +"Bb"z16|"Bb"z16|"F7"z16|"F7"z16| +V: Ins +d'4d'2c'2d'4d'2c'2|d'4d'2c'2d'2c'2b2d'2|c'4c'2b2c'4c'2b2|c'4c'2b2c'2b2a2c'2| +V: Vocal +"Bb"z16|"Bb"z16|"F7"z16|"Bb"z16| +V: Ins +d'4d'2c'2d'4d'2c'2|d'4d'2c'2d'2c'2b2d'2|c'4c'2b2c'2b2a2c'2|b16| +V: Vocal +"Bb"z16|"Bb"z16|"Bb"z16| +V: Ins +Z3| diff --git a/examples/yue2_style_change/style_change.py b/examples/yue2_style_change/style_change.py new file mode 100644 index 000000000..82061efc7 --- /dev/null +++ b/examples/yue2_style_change/style_change.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Change a YuE2 song's style part-way through by editing its semantic history. + +Run from the repository root; unknown options go to audiocpp_cli unchanged: + + python3 examples/yue2_style_change/style_change.py --backend cuda + +Needs only the Python standard library. Steps whose output directory already +exists are skipped, so delete a directory under --out to redo that step. +""" +import argparse, fractions, json, pathlib, subprocess + +HERE = pathlib.Path(__file__).resolve().parent +FPS = 25 # semantic frames per second of audio +LEAD = 35 # the audio runs about this many frames ahead of the written score + +ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) +ap.add_argument('--cli', default='build/bin/audiocpp_cli', help='audiocpp_cli binary (default: %(default)s)') +ap.add_argument('--model', default='models/Yue2-3B-GGUF', help='YuE2 model directory (default: %(default)s)') +ap.add_argument('--lyrics', default=str(HERE / 'lyrics.txt')) +ap.add_argument('--score', default=str(HERE / 'score.abc'), help='score both styles are sung from') +ap.add_argument('--new-score', action='store_true', help='write a fresh score instead (stop_after=abc)') +ap.add_argument('--style-a', default='animated film musical, storybook, minor key, string section answering the vocal, ' + 'call and response, pizzicato strings, female vocal, dramatic, storytelling') +ap.add_argument('--style-b', default='brutal death metal, blast beats, down-tuned chugging guitars, ' + 'guttural growled male vocal, shrieked backing screams, horror, relentless') +ap.add_argument('--cut', type=float, help='second at which the style changes (default: the second chorus)') +ap.add_argument('--seed', type=int, default=3) +ap.add_argument('--out', default='yue2_style_change_out') +args, cli_options = ap.parse_known_args() + +out = pathlib.Path(args.out) +lyrics = pathlib.Path(args.lyrics).read_text(encoding='utf-8') + + +def run(name, style, **options): + """One audiocpp_cli request. Returns the directory holding its artifacts.""" + where = out / name + if not where.exists(): + cmd = [args.cli, '--task', 'gen', '--family', 'yue2', '--model', args.model, '--lyrics', lyrics, + '--seed', str(args.seed), '--out-dir', str(where), '--request-option', 'style=' + style] + if 'stop_after' not in options: + cmd += ['--out', str(where) + '.wav'] + for key, value in options.items(): + cmd += ['--request-option', '%s=%s' % (key, value)] + print('==', name, flush=True) + subprocess.run(cmd + cli_options, check=True) + return where + + +def sections(score): + """(start second, name) of each section, from the bar counts of the vocal voice""" + bar, quarter = 4.0, 0.5 # quarter notes per bar, seconds per quarter note + found, seconds, name, vocal = [], 0.0, None, False + for line in pathlib.Path(score).read_text().splitlines(): + if line.startswith('M:'): + bar = float(fractions.Fraction(line[2:].strip())) * 4 + elif line.startswith('Q:'): + unit, tempo = line[2:].split('=') + quarter = 60 / (float(tempo) * float(fractions.Fraction(unit)) * 4) + elif line.startswith('% '): + name = line[2:] + elif line.startswith('V:'): + vocal = line.startswith('V: Vocal') + elif vocal and '|' in line and not line.startswith('w:'): + if name: + found.append((seconds, name)) + name = None + seconds += line.count('|') * bar * quarter + return found + + +def tokens(where): + return json.loads((where / 'semantic.json').read_text()) + + +def save(name, frames): + out.mkdir(parents=True, exist_ok=True) + path = out / name + path.write_text(json.dumps(frames)) + return path + + +def lag(song, take, at, span=30 * FPS, reach=4 * FPS): + """How many frames later `take` plays what `song` plays before `at`. + + Two renders of one score do not keep the same clock. At the right lag they share + a few percent of identical tokens, at any other lag almost none. + """ + def hits(shift): + return sum(1 for i in range(max(at - span, -shift, 0), at) if i + shift < len(take) and song[i] == take[i + shift]) + best = max(range(-reach, reach + 1), key=hits) + return best if hits(best) >= 8 else 0 + + +# 1. One score: the bundled one, or a fresh one. A score decides how far a second +# style can move away from the first, so a fresh one is a gamble. +score = run('1_score', args.style_a, stop_after='abc') / 'score.abc' if args.new_score else pathlib.Path(args.score) +starts = sections(score) +print('sections:', ', '.join('%s %d s' % (name, second) for second, name in starts)) +if args.cut is None: + choruses = [second for second, name in starts if name == 'chorus'] + args.cut = choruses[1] if len(choruses) > 1 else starts[len(starts) // 2][0] +cut = round(args.cut * FPS) - LEAD + +# 2. The same score sung in both styles; tokens only, no audio yet. +only_tokens = dict(abc_file=score, stop_after='semantic') +song = tokens(run('2_take_a', args.style_a, **only_tokens)) +take_b = tokens(run('2_take_b', args.style_b, **only_tokens)) + +# (where the history is edited, how much of the real song is left at its end), frames. +# The first edit is early and long: the song stays in style A but plays a lead-in. +# The second is the change itself. +edits = [(cut - 10 * FPS, 30 * FPS), (cut, 5 * FPS)] +if edits[0][0] <= edits[0][1] + 4 * FPS or cut >= min(len(song), len(take_b)) - 4 * FPS: + raise SystemExit('--cut must be later than 46 s and inside both takes (%d s and %d s)' + % (len(song) // FPS, len(take_b) // FPS)) + +# 3. At each edit the model is handed style B's take as its past, with the last +# `keep` frames of the real song on the end, and asked what comes next. What it +# writes replaces the song from there on. A leg stops where the next edit starts. +for index, (at, keep) in enumerate(edits): + shift = lag(song, take_b, at) + print('edit at %.1f s: style B runs %+d frames against the song' % (at / FPS, shift)) + history = save('3_history_%d.json' % index, take_b[:at + shift - keep] + song[at - keep:at]) + bounds = {} + if index + 1 < len(edits): + stop = edits[index + 1][0] + shift + bounds = dict(semantic_min_tokens=stop, semantic_max_tokens=stop) + leg = tokens(run('3_leg_%d' % index, args.style_b, semantic_prefix_file=history, **only_tokens, **bounds)) + song = song[:at] + leg[at + shift:] + +# 4. Render the finished stream. The prefix is the whole song, so nothing is sampled. +final = run('4_song', args.style_a, abc_file=score, semantic_prefix_file=save('4_song.json', song), + semantic_min_tokens=len(song), semantic_max_tokens=len(song)) +print('done: %s.wav, %d s, style change at %.1f s' % (final, len(song) // FPS, cut / FPS)) diff --git a/include/engine/models/yue2/ar_runtime.h b/include/engine/models/yue2/ar_runtime.h index 206384123..c2484bd09 100644 --- a/include/engine/models/yue2/ar_runtime.h +++ b/include/engine/models/yue2/ar_runtime.h @@ -45,14 +45,16 @@ class Yue2ArRuntime { std::vector generate( const std::vector & prefix, const Yue2ArSamplingWindow & window, - uint64_t seed); + uint64_t seed, + const std::vector & forced = {}); std::vector generate_cfg( const std::vector & positive_prefix, const std::vector & negative_prefix, const Yue2ArSamplingWindow & window, float guidance_scale, - uint64_t seed); + uint64_t seed, + const std::vector & forced = {}); runtime::TransformerKVState prefill_state(const std::vector & tokens); Yue2ArDevicePrefixState prefill_device_state(const std::vector & tokens); diff --git a/include/engine/models/yue2/types.h b/include/engine/models/yue2/types.h index d2df7b592..c3b7edfa6 100644 --- a/include/engine/models/yue2/types.h +++ b/include/engine/models/yue2/types.h @@ -83,6 +83,7 @@ struct Yue2Request { Yue2CotMode cot = Yue2CotMode::Full; Yue2StopAfter stop_after = Yue2StopAfter::Audio; std::string abc; + std::vector semantic_prefix; std::vector nar_noise; bool export_semantic = false; uint64_t seed = 1234; diff --git a/model_specs/yue2.json b/model_specs/yue2.json index d0bfc18f0..54d0c0142 100644 --- a/model_specs/yue2.json +++ b/model_specs/yue2.json @@ -88,6 +88,18 @@ "description": "Path to an external ABC score file for melody/full routes.", "required": false }, + { + "name": "semantic_prefix", + "type": "string", + "description": "JSON array of semantic codec indices to force as the start of the music stream.", + "required": false + }, + { + "name": "semantic_prefix_file", + "type": "string", + "description": "Path to a JSON array of semantic codec indices to force as the start of the music stream.", + "required": false + }, { "name": "nar_noise_file", "type": "string", diff --git a/src/models/yue2/ar_runtime.cpp b/src/models/yue2/ar_runtime.cpp index 5eb4bb9a0..81c101536 100644 --- a/src/models/yue2/ar_runtime.cpp +++ b/src/models/yue2/ar_runtime.cpp @@ -660,10 +660,14 @@ struct Yue2ArRuntime::Impl { std::vector generate( const std::vector & prefix, const Yue2ArSamplingWindow & window, - uint64_t seed) { + uint64_t seed, + const std::vector & forced) { if (prefix.empty()) { throw std::runtime_error("Yue2 AR prefix must not be empty"); } + if (static_cast(forced.size()) > window.max_tokens) { + throw std::runtime_error("Yue2 AR forced tokens exceed the sampling window"); + } const bool compact_semantic = is_semantic_window(window); const bool compact_abc = is_abc_window(window); ensure_generation_runtime(false, compact_semantic, compact_abc); @@ -673,13 +677,24 @@ struct Yue2ArRuntime::Impl { auto cache_steps_for = [](int64_t prefix_tokens, int64_t remaining_tokens) { return prefix_tokens + std::min(remaining_tokens, kArDecodeChunkTokens); }; + std::vector emitted; + emitted.reserve(static_cast(window.max_tokens)); + emitted.insert(emitted.end(), forced.begin(), forced.end()); + std::vector forced_prefix; + if (!forced.empty()) { + forced_prefix.reserve(prefix.size() + forced.size()); + forced_prefix.insert(forced_prefix.end(), prefix.begin(), prefix.end()); + forced_prefix.insert(forced_prefix.end(), forced.begin(), forced.end()); + engine::debug::timing_log_scalar("yue2.ar.generate.forced_tokens", forced.size()); + } + const auto & start_prefix = forced.empty() ? prefix : forced_prefix; const auto prefill_start = Clock::now(); auto prefill = active_runtime->prefill_tokens_into_decode_cache( - prefix, - cache_steps_for(static_cast(prefix.size()), window.max_tokens)); + start_prefix, + cache_steps_for( + static_cast(start_prefix.size()), + window.max_tokens - static_cast(emitted.size()))); engine::debug::timing_log_scalar("yue2.ar.generate.prefill_ms", engine::debug::elapsed_ms(prefill_start)); - std::vector emitted; - emitted.reserve(static_cast(window.max_tokens)); std::mt19937 rng(static_cast(seed)); Yue2SamplerScratch scratch; engine::modules::QwenCausalDecodeStepResult decode_result; @@ -689,7 +704,7 @@ struct Yue2ArRuntime::Impl { double decode_ms = 0.0; double refill_prefill_ms = 0.0; int64_t refill_count = 0; - for (int64_t step = 0; step < window.max_tokens; ++step) { + for (int64_t step = static_cast(emitted.size()); step < window.max_tokens; ++step) { const auto sample_start = Clock::now(); const int32_t token = compact_semantic ? sample_semantic_token(decode_result.logits, emitted, window, rng, scratch) : @@ -742,41 +757,58 @@ struct Yue2ArRuntime::Impl { const std::vector & negative_prefix, const Yue2ArSamplingWindow & window, float guidance_scale, - uint64_t seed) { + uint64_t seed, + const std::vector & forced) { if (guidance_scale == 1.0F) { - return generate(positive_prefix, window, seed); + return generate(positive_prefix, window, seed, forced); + } + if (static_cast(forced.size()) > window.max_tokens) { + throw std::runtime_error("Yue2 AR forced tokens exceed the sampling window"); } const bool compact_semantic = is_semantic_window(window); const bool compact_abc = is_abc_window(window); ensure_generation_runtime(false, compact_semantic, compact_abc); auto & positive_runtime = compact_semantic ? semantic_runtime : (compact_abc ? abc_runtime : runtime); const auto total_start = Clock::now(); - engine::debug::timing_log_scalar("yue2.ar.cfg.positive_prefix_tokens", positive_prefix.size()); - engine::debug::timing_log_scalar("yue2.ar.cfg.negative_prefix_tokens", negative_prefix.size()); + auto append_forced = [&forced](const std::vector & tokens) { + std::vector out; + out.reserve(tokens.size() + forced.size()); + out.insert(out.end(), tokens.begin(), tokens.end()); + out.insert(out.end(), forced.begin(), forced.end()); + return out; + }; + const auto positive_tokens = append_forced(positive_prefix); + const auto negative_tokens = append_forced(negative_prefix); + engine::debug::timing_log_scalar("yue2.ar.cfg.positive_prefix_tokens", positive_tokens.size()); + engine::debug::timing_log_scalar("yue2.ar.cfg.negative_prefix_tokens", negative_tokens.size()); + if (!forced.empty()) { + engine::debug::timing_log_scalar("yue2.ar.cfg.forced_tokens", forced.size()); + } const auto positive_prefill_start = Clock::now(); - auto positive = positive_runtime->prefill_tokens(positive_prefix); + auto positive = positive_runtime->prefill_tokens(positive_tokens); engine::debug::timing_log_scalar("yue2.ar.cfg.prefill_positive_ms", engine::debug::elapsed_ms(positive_prefill_start)); const auto negative_prefill_start = Clock::now(); - auto negative = positive_runtime->prefill_tokens(negative_prefix); + auto negative = positive_runtime->prefill_tokens(negative_tokens); engine::debug::timing_log_scalar("yue2.ar.cfg.prefill_negative_ms", engine::debug::elapsed_ms(negative_prefill_start)); const auto start_decode_start = Clock::now(); const int64_t cache_steps = std::max( - static_cast(positive_prefix.size()), - static_cast(negative_prefix.size())) + - window.max_tokens; + static_cast(positive_tokens.size()), + static_cast(negative_tokens.size())) + + window.max_tokens - static_cast(forced.size()); positive_runtime->start_decode_tokens_batched( make_cfg_batched_state(positive.state, negative.state), cache_steps); engine::debug::timing_log_scalar("yue2.ar.cfg.start_decode_ms", engine::debug::elapsed_ms(start_decode_start)); std::vector emitted; emitted.reserve(static_cast(window.max_tokens)); + emitted.insert(emitted.end(), forced.begin(), forced.end()); std::mt19937 rng(static_cast(seed)); Yue2SamplerScratch scratch; std::vector logits(positive.logits.size(), 0.0F); double sample_ms = 0.0; double decode_batched_ms = 0.0; - for (int64_t step = 0; step < window.max_tokens; ++step) { + for (int64_t step = static_cast(emitted.size()); step < window.max_tokens; ++step) { if (positive.logits.size() != negative.logits.size()) { throw std::runtime_error("Yue2 CFG logits size mismatch"); } @@ -953,8 +985,9 @@ Yue2ArRuntime::~Yue2ArRuntime() = default; std::vector Yue2ArRuntime::generate( const std::vector & prefix, const Yue2ArSamplingWindow & window, - uint64_t seed) { - return impl_->generate(prefix, window, seed); + uint64_t seed, + const std::vector & forced) { + return impl_->generate(prefix, window, seed, forced); } std::vector Yue2ArRuntime::generate_cfg( @@ -962,8 +995,9 @@ std::vector Yue2ArRuntime::generate_cfg( const std::vector & negative_prefix, const Yue2ArSamplingWindow & window, float guidance_scale, - uint64_t seed) { - return impl_->generate_cfg(positive_prefix, negative_prefix, window, guidance_scale, seed); + uint64_t seed, + const std::vector & forced) { + return impl_->generate_cfg(positive_prefix, negative_prefix, window, guidance_scale, seed, forced); } runtime::TransformerKVState Yue2ArRuntime::prefill_state(const std::vector & tokens) { diff --git a/src/models/yue2/pipeline.cpp b/src/models/yue2/pipeline.cpp index fb83f44ac..524327088 100644 --- a/src/models/yue2/pipeline.cpp +++ b/src/models/yue2/pipeline.cpp @@ -80,6 +80,18 @@ std::vector codec_from_semantic_tokens(const std::vector & tok return out; } +std::vector semantic_tokens_from_codec(const std::vector & codec) { + std::vector out; + out.reserve(codec.size()); + for (const int32_t index : codec) { + if (index < 0 || index >= kCodecSize) { + throw std::runtime_error("Yue2 semantic prefix codec index is out of range"); + } + out.push_back(kCodecOffset + index); + } + return out; +} + Yue2ArSamplingWindow abc_window(const Yue2GenerationConfig & generation) { return Yue2ArSamplingWindow{ 0, @@ -162,6 +174,18 @@ class Yue2PipelineRuntime::Impl { Yue2SemanticResult out; out.plan = std::move(plan); generate_plan_abc(request, out.plan); + const auto forced = semantic_tokens_from_codec(request.semantic_prefix); + if (!forced.empty() && + static_cast(forced.size()) >= request.generation.semantic.max_tokens) { + // The prefix already fills the window: nothing would be sampled, so + // the AR prefill and its logits are skipped. stop_after=audio still + // prefills the stream once, in the NAR stage, for its conditioning. + out.tokens = forced; + out.truncated = true; + engine::debug::timing_log_scalar("yue2.semantic.tokens", out.tokens.size()); + engine::debug::timing_log_scalar("yue2.semantic.truncated", out.truncated); + return out; + } ensure_ar(); const auto neg = negative_prefix(request, tokenizer, out.plan.abc_ids); out.tokens = ar->generate_cfg( @@ -169,7 +193,8 @@ class Yue2PipelineRuntime::Impl { neg, semantic_window(request.generation), request_guidance_scale(request), - request.seed); + request.seed, + forced); out.truncated = static_cast(out.tokens.size()) >= request.generation.semantic.max_tokens; engine::debug::timing_log_scalar("yue2.semantic.tokens", out.tokens.size()); engine::debug::timing_log_scalar("yue2.semantic.truncated", out.truncated); @@ -286,7 +311,8 @@ class Yue2PipelineRuntime::Impl { << " context=" << request.generation.context << " abc=" << (request.abc.empty() ? (request.cot == Yue2CotMode::Off ? "none" : "generated") : "provided") - << " nar_noise=" << (request.nar_noise.empty() ? "generated" : "provided"); + << " nar_noise=" << (request.nar_noise.empty() ? "generated" : "provided") + << " semantic_prefix_frames=" << request.semantic_prefix.size(); engine::debug::trace_log_scalar("yue2.request", settings.str()); for (const bool abc : {true, false}) { if (abc && (request.cot == Yue2CotMode::Off || !request.abc.empty())) { diff --git a/src/models/yue2/request.cpp b/src/models/yue2/request.cpp index 4f5b882f9..6951c1c9d 100644 --- a/src/models/yue2/request.cpp +++ b/src/models/yue2/request.cpp @@ -2,12 +2,15 @@ #include "engine/framework/io/filesystem.h" #include "engine/framework/io/binary.h" +#include "engine/framework/io/json.h" #include "engine/framework/runtime/options.h" #include +#include #include #include #include +#include namespace engine::models::yue2 { namespace { @@ -52,6 +55,61 @@ std::string abc_from_options(const std::unordered_map return {}; } +std::vector parse_semantic_prefix(const std::string & text, const std::string & source) { + engine::io::json::Value root; + try { + root = engine::io::json::parse(text); + } catch (const std::exception & error) { + throw std::runtime_error("Yue2 " + source + " must be a JSON array of codec indices: " + error.what()); + } + if (!root.is_array()) { + throw std::runtime_error("Yue2 " + source + " must be a JSON array of codec indices"); + } + const auto & items = root.as_array(); + if (items.empty()) { + throw std::runtime_error("Yue2 " + source + " must contain at least one codec index"); + } + std::vector out; + out.reserve(items.size()); + for (size_t index = 0; index < items.size(); ++index) { + const auto & item = items[index]; + if (!item.is_number()) { + throw std::runtime_error( + "Yue2 " + source + " entry " + std::to_string(index) + " is not a codec index"); + } + int64_t value = 0; + try { + value = item.as_i64(); + } catch (const std::exception &) { + throw std::runtime_error( + "Yue2 " + source + " entry " + std::to_string(index) + " is not an integer"); + } + if (value < 0 || value >= kCodecSize) { + throw std::runtime_error( + "Yue2 " + source + " entry " + std::to_string(index) + " is out of range [0," + + std::to_string(kCodecSize) + ")"); + } + out.push_back(static_cast(value)); + } + return out; +} + +std::vector semantic_prefix_from_options(const std::unordered_map & options) { + if (const auto prefix = runtime::find_option(options, {"semantic_prefix"})) { + if (!prefix->empty()) { + return parse_semantic_prefix(*prefix, "semantic_prefix"); + } + } + if (const auto prefix_file = runtime::find_option(options, {"semantic_prefix_file"})) { + const std::filesystem::path path(*prefix_file); + if (!engine::io::is_existing_file(path)) { + throw std::runtime_error("Yue2 semantic_prefix_file does not exist: " + path.string()); + } + return parse_semantic_prefix(engine::io::read_text_file(path), "semantic_prefix_file"); + } + return {}; +} + void apply_options( Yue2Request & out, const std::unordered_map & options) { @@ -135,6 +193,15 @@ void apply_options( throw std::runtime_error("Yue2 stop_after=abc generates no score when abc or abc_file is supplied"); } } + out.semantic_prefix = semantic_prefix_from_options(options); + if (!out.semantic_prefix.empty()) { + if (static_cast(out.semantic_prefix.size()) > out.generation.semantic.max_tokens) { + throw std::runtime_error("Yue2 semantic prefix is longer than semantic_max_tokens"); + } + if (out.cot != Yue2CotMode::Off && out.abc.empty()) { + throw std::runtime_error("Yue2 semantic prefix requires abc or abc_file unless cot=off"); + } + } if (const auto nar_noise_file = runtime::find_option(options, {"nar_noise_file"})) { const std::filesystem::path path(*nar_noise_file); if (!engine::io::is_existing_file(path)) { diff --git a/tools/audiocpp_cli/audiocpp_cli_path_cases.json b/tools/audiocpp_cli/audiocpp_cli_path_cases.json index 5e9fa6d0d..1230232c6 100644 --- a/tools/audiocpp_cli/audiocpp_cli_path_cases.json +++ b/tools/audiocpp_cli/audiocpp_cli_path_cases.json @@ -1425,6 +1425,95 @@ } ] }, + { + "id": "yue2_semantic_prefix_continue", + "coverage": "YuE2 continuation from a forced semantic prefix on the direct route", + "family": "yue2", + "model": "models/YuE2-3B-Q8_0-GGUF", + "task": "gen", + "mode": "offline", + "outputs": [ + "audio" + ], + "requests": [ + { + "id": "semantic_prefix_continue", + "text": "[Verse]\nA quiet harbor wakes up slow.\nThe lantern dims, the tide runs low.\nWe count the boats along the line.\nThe morning keeps its steady time.\n[Chorus]\nCarry on, carry on.\nHold the note until it is gone.", + "seed": 20260914, + "num_inference_steps": 8, + "options": { + "cfg_scale": 1.0, + "style": "English, folk pop, acoustic guitar, soft brushes, warm male vocal, relaxed tempo", + "cot": "off", + "semantic_prefix": [ + 12046, + 8433, + 16287, + 11138, + 6563, + 22418, + 9529, + 23781, + 7955, + 1103, + 6583, + 4347, + 17256, + 9470, + 4579, + 28211 + ], + "semantic_min_tokens": 192, + "semantic_max_tokens": 640 + } + } + ] + }, + { + "id": "yue2_semantic_prefix_exact", + "coverage": "YuE2 forced semantic prefix that fills the window: the AR stage is skipped and the given frames are exported as they came", + "family": "yue2", + "model": "models/YuE2-3B-Q8_0-GGUF", + "task": "gen", + "mode": "offline", + "outputs": [ + "artifact" + ], + "requests": [ + { + "id": "semantic_prefix_exact", + "text": "[Verse]\nA quiet harbor wakes up slow.\nThe lantern dims, the tide runs low.\n[Chorus]\nCarry on, carry on.\nHold the note until it is gone.", + "seed": 20260914, + "num_inference_steps": 8, + "options": { + "cfg_scale": 1.0, + "style": "English, folk pop, acoustic guitar, soft brushes, warm male vocal, relaxed tempo", + "cot": "off", + "stop_after": "semantic", + "semantic_prefix": [ + 12046, + 8433, + 16287, + 11138, + 6563, + 22418, + 9529, + 23781, + 7955, + 1103, + 6583, + 4347, + 17256, + 9470, + 4579, + 28211 + ], + "semantic_min_tokens": 16, + "semantic_max_tokens": 16 + } + } + ] + }, { "id": "irodori_tts_500m_emoji_style_clone", "coverage": "Irodori-TTS 500M no-reference emoji/style requests and reference clone path in one session",