SwarmUI-HartsyInference.csproj

<Project Sdk="Microsoft.NET.Sdk.Web">
    <PropertyGroup>
        <AssemblyName>SwarmExtensionHartsyInference</AssemblyName>
        <!-- Required by the ControlNet annotators / Wan-Animate preprocessors (Generation/*Preprocessor.cs,
             ControlVideoDecoder) — they fill tensor storage through byte*/float* without a managed copy. -->
        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
        <!-- Make the floating HartsyInference 'alpha.*' reference (below) ACTUALLY resolve to the
             latest published version on every build. Without this, NuGet serves the flat-container
             version index from its local HTTP cache (~30 min TTL), so a freshly-published alpha.N
             stays invisible and the build silently sticks on a stale cached version. SwarmUI's
             ExtensionsManager.BuildExtension runs a plain `dotnet build` (implicit restore, with no
             cache-bypassing flag), so we force the live feed query here at the project level.
             Trade-off: extension builds now require network access to nuget.org (no offline restore
             of a NEW floating version; already-resolved versions in the global-packages folder are
             unaffected). Pin Version below to a specific alpha.N if you ever need an offline build. -->
        <RestoreNoHttpCache>true</RestoreNoHttpCache>
        <!-- Engine source toggle (same pattern as SwarmUI-LLMAssistant): false = published NuGet;
             true = prebuilt local HartsyInference DLLs. MUST default false — this extension is shipped to
             end users who only have nuget.org, so the pin below must always be a REAL published version.
             Only flip true (via -p:UseLocalHartsy=true) for local engine dev against unpublished DLLs. -->
        <UseLocalHartsy Condition="'$(UseLocalHartsy)' == ''">false</UseLocalHartsy>
        <HartsyRepo Condition="'$(HartsyRepo)' == ''">$(MSBuildThisFileDirectory)../../../../HartsyInference</HartsyRepo>
        <!-- Cli bin is the full engine closure: every HartsyInference.* assembly + Google.Protobuf +
             Microsoft.ML.Tokenizers, all in one folder. -->
        <HartsyLocalBin>$(HartsyRepo)/src/HartsyInference.Cli/bin/Release/net8.0</HartsyLocalBin>
    </PropertyGroup>
    <Import Project="../../SwarmUI.extension.props" />

    <!-- SwarmUI.extension.props sets TargetFramework=net8.0 (matching the net8.0 host) and
         CopyLocalLockFileAssemblies=false. KEEP the inherited net8.0 TFM — the engine is multi-target
         (net8.0 + net10.0), so net8.0 pulls the net8.0 engine assets that load in the net8.0 host. Overriding
         to net10.0 pulls net10.0 engine DLLs (System.Runtime 10.0.0.0) that the host's per-extension
         AssemblyLoadContext cannot resolve → ReflectionTypeLoadException. Only override
         CopyLocalLockFileAssemblies — the engine's private DLLs (HartsyInference.*, Microsoft.ML.Tokenizers,
         Google.Protobuf) must land in THIS extension's output folder so the ALC can resolve them; we then
         exclude the host-provided packages below. -->
    <PropertyGroup>
        <!-- Inherit net8.0 from SwarmUI.extension.props (matches the net8.0 host). Do NOT override to net10.0:
             the engine is multi-target, so net8.0 pulls the net8.0 engine assets that load in the net8.0 host.
             A net10.0 extension → net10.0 engine DLLs (System.Runtime 10.0.0.0) → the host ALC can't resolve. -->
        <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
    </PropertyGroup>

    <!-- The pure-C#/.NET HartsyInference inference engine. The meta-package pulls every backend
         (CPU/CUDA/Vulkan) and modality assembly (Diffusion/Audio/Video/Vision/…) plus its private
         deps (Microsoft.ML.Tokenizers → Google.Protobuf). Its build targets auto-copy the CUDA PTX
         (→ Ptx/) and Vulkan SPIR-V (→ Spirv/) kernels into our output, exactly where the backend
         resolves them next to the extension DLL. https://www.nuget.org/packages/HartsyInference -->
    <ItemGroup Condition="'$(UseLocalHartsy)' != 'true'">
        <!-- Pinned prerelease for a reproducible build. The publish-nuget.yml workflow ships
             1.0.0-alpha.<run_number> on each push to main; bump this to the EXACT version that CI
             published (check the run's "Publishing version:" log line) whenever you ship engine changes.
             alpha.25 = lazy host-buffer allocation (GPU activations no longer pay host malloc+memset).
             alpha.33 = Boogu-Image pipeline + Qwen3-VL vision/multimodal encoders.
             alpha.35 = first version with Krea 2 (transformer + pipeline + converter + Qwen3_VL_4B preset).
             alpha.36 = Krea 2 keys.
             alpha.38 = CUDA LeakyRelu + AdaInstanceNorm1d kernels (Kokoro/StyleTTS2 runs fully on GPU).
                        (alpha.37 was burned by a pre-fix pack; bumped to dodge the immutable NuGet cache.)
             alpha.43 = query-tiled CUDA SDPA for long sequences (Wan-Video full-res self-attention no
                        longer materializes the ~49GB score matrix), Wan VAE + variant work, LTX fixes.
             alpha.44 = WanVideoConfig.CfgRescale + MoE expert swap + umT5/T5 AttentionScale=1.0 fix
                        (the flat-video root cause).
             alpha.45 = engine-side work (TTS perf, Yue vocal decoder); the "HartsyInference" META-PACKAGE
                        itself was never actually published at this version — its automated publish step
                        didn't exist yet (see alpha.46 note). Sub-packages only.
             alpha.46 = Boogu-Image DiT residency fix (EvictResidentWeights) + the meta-package publish
                        gap fixed: "HartsyInference" is now packed via a real project
                        (src/HartsyInference.Meta) wired into publish-nuget.yml, instead of a manual,
                        easily-forgotten `dotnet pack`+push that had been stuck at alpha.44 for weeks.
             alpha.49 = Kyutai TTS (Delayed-Streams Modeling) fully intelligible e2e — cross-attention
                        voice source now includes the learnt-padding speaker slots (the missing-rows bug
                        that produced non-speech) + text-token sampling for word pacing; depformer
                        weights-per-step projections made device-resident (63.0 → 9.7 s on a 3060).
             (Note: the alpha.* line was reset to 2.0.0-alpha.N during the Engine-architecture rewrite;
             the alpha.25-alpha.49 history above predates that reset and refers to the old 1.0.0 line.)
             2.0.0-alpha.3 = latest published on nuget.org as of 2026-07-25; predates the same-day
             commit that made TextService.FindMmproj public (see SwarmUI-LLMAssistant's HartsyLocalLLMProvider).
             Pinned here to 2.0.0-alpha.5, ahead of the engine repo's publish of that version — restore
             will 404 until that push lands on nuget.org, so merge the engine bump FIRST. alpha.5 is the
             first version carrying InferenceEngine(selector, ordinal), which is what makes the GPU_ID
             setting real; pinning any earlier version will not compile this extension.
             NEVER pin a "-local" suffix here: those are locally-packed builds that live only on the
             dev machine's nuget.config feed and are NOT on nuget.org, so any downloaded copy of this
             extension fails restore with NU1301 (the "missing local dll"-style error). Always pin an
             EXACT version that appears in the flat-container index above.
             2.0.0-alpha.7 = the generated-audio return path: IVideoService.GenerateAsync returns
             VideoGenerationResult (frames + AudioBuffer) instead of IAsyncEnumerable&lt;VideoFrame&gt;.
             This extension REQUIRES that shape — GenerateVideo does not compile against alpha.6.
             alpha.7 was tagged but never landed on nuget.org, so do NOT pin it.
             2.0.0-alpha.8 = alpha.7 plus the audio/video length fit (a short soundtrack was making
             ffmpeg -shortest drop the clip's final frame).
             2.0.0-alpha.9 = device-gate/multi-GPU placement support: EngineOptions.LowVram (per-backend
             low-VRAM policy, instead of the process-wide HARTSY_LOWVRAM env var) and EngineOptions.Placement
             (PlacementConfig, multi-device component/shard placement), the InferenceEngine(selector,
             ordinal, options) 3-arg constructor, InferenceEngine.ComputeBackend, and BackendFactory.Validate
             (selector validation) — all already committed on main but never published past alpha.8, which is
             what broke this extension's build against the published package.
             2.0.0-alpha.10 = AudioModelCache is now category-aware (tts/stt/music/clone/fx/misc) and defaults
             its cache root to {HARTSYINFERENCE_MODELS}/audio instead of an independent ~/.cache/hartsyinference
             tree, so audio downloads land in the same Models/ folder as everything else instead of a separate
             flat cache — part of the Models-folder consolidation (see hartsy-multigpu-overhaul memory / the
             2026-08-03 migration). Pinned here ahead of publish, same as the alpha.5 precedent above — restore
             will 404 until the engine repo's commit pushes and publish-nuget.yml lands it on nuget.org. -->
        <!-- 2.0.0-alpha.11 = multi-GPU production pass: DiT sharding for Qwen-Image/MiniMax-H3/Flux/Chroma/
             HunyuanImage, TE/VAE placement for the remaining pipelines (PlacementConfig.VaeDevice now settable),
             CFG-parallel observability ([CfgParallel] decisions) + Flux recipe forwarding fix, catalog-aware
             `hartsy pull`. -->
        <!-- 2.0.0-alpha.12 = MiniMax-H3 conditioning + LoRA. REQUIRED by this extension's Reference Images /
             Reference Audio params: VideoRequest gains ReferenceImages/ReferenceVideos/ReferenceAudios (and the
             ReferenceVideo record), which do not exist in alpha.11, so those params will not compile against it.
             Also carries fl2va keyframe conditioning (InitImage/VideoEndFrame now drive first/last-frame video),
             both H3 VAE encoder halves, LoRA wiring for the H3 recipe (first video model with LoRA end to end),
             and a CLI-drivable minimax-h3 catalog entry with assets + sha256.
             Restore 404s until the alpha.12 publish lands — by design (alpha.4/.5/.10/.11 precedent). -->
        <!-- 2.0.0-alpha.13 was committed and its CI publish did NOT land (nupkg 404 on nuget.org), so the number is
             burned rather than reused — same treatment as alpha.37/alpha.45 above. Everything it described ships in
             alpha.14 below.
             2.0.0-alpha.14 = video LoRA actually works, plus the multi-GPU fix.
             (a) GetOrConstructVideoRecipe now takes the VideoRequest. It previously took only the ModelSpec, so
                 RecipeContext.Loras AND .Components were always null for EVERY video family — a selected LoRA
                 silently produced bit-identical output and per-request component overrides were dropped. The video
                 pipeline cache key now includes the LoRA stack, so a LoRA request cannot reuse the un-merged pipeline.
             (b) MiniMaxH3Recipe declares VideoFeatures.Lora, which this extension's honesty guard reads to decide
                 whether a LoRA selection is allowed on a video model. Without it the guard refuses every video LoRA.
             (c) GpuTransferHelper: a host tensor >1 MB uploaded by a SECOND backend was allocated in the FIRST
                 backend's CUDA context (the demote hook switches context and never restores), so any DiT-sharded run
                 touching one died with CUDA_ERROR_INVALID_VALUE — which H3's fl2va rope tables hit. Sharded step time
                 also drops ~2.9x now that those tensors can be promoted on the second card.
             alpha.15 adds MiniMax-H3 reference VIDEOS (a 2-frame vision block is gridT=1, so it costs a still
             image's tokens) and LoRA on the fp8 build (dequant -> merge in F32 -> requantize with a recomputed
             absmax/448 scale, ComfyUI's approach, so the native fp8 GEMM survives). Reference videos need the
             "H3 Reference Videos" param registered below.
             Restore 404s until the alpha.15 publish lands — by design (alpha.4/.5/.10/.11 precedent). -->
        <!-- 2.0.0-alpha.17 = REQUIRED; this extension does not COMPILE against alpha.15/.16, which predate several
             engine APIs it already programs against: VideoFeatures.ReferenceImages/ReferenceVideos/ReferenceAudios
             and .DrivingVideo, VideoRequest.DrivingVideo/DrivingPoseVideo/DrivingFaceVideo/DrivingAutoPreprocess,
             MusicRequest.TopP/RepetitionPenalty, RequestExtras.ReduxStyleModel, and WanVideoRecipe.SupportsFor.
             On top of those it carries the MiniMax-H3 VRAM/OOM pass this extension's UX depends on:
             (a) A per-request pre-flight that refuses an impossible geometry BEFORE allocating, naming the byte
                 requirement, which card fell short, and the longest clip that would fit at that resolution — the
                 text GenerateLive quotes into its readable SwarmReadableErrorException. Without it an oversized
                 request OOMs mid-forward and surfaces as "Something went wrong".
             (b) Chunked attention now scatters k/v into their full buffers instead of concatenating chunk lists,
                 cutting the pass-1 peak from 5x to 3x seq*inner*F32 — a 141-frame sharded run that OOMed now
                 completes, bit-identical.
             (c) Geometry guard: requests above the trained area are scaled down preserving aspect, and clips past
                 the ~15 s trained envelope warn instead of silently degrading.
             (d) TE/VAE placement honoured for MiniMax-H3, so TextEncoderGpuId/VaeGpuId finally do something on it;
                 off-primary components keep their weights resident between generations.
             (e) LoRA: the published H3 Turbo LoRA (bare checkpoint keys, no transformer. prefix) was rejected at
                 load as an unknown format; it now loads, and a delta whose shape does not match the checkpoint is
                 skipped with a named warning instead of being handed to backend.Add.
             Restore 404s until the engine's alpha.17 publish lands — by design (alpha.4/.5/.10/.11 precedent):
             merge the engine bump FIRST, then this pin. -->
        <!-- 2.0.0-alpha.26 shipped the seamless-tiling rollout but 18 of 25 families dropped the flag in
             their hand-built inner requests (silent no-op); 2.0.0-alpha.27 forwards it everywhere and is the
             version verified end-to-end (Flux wrap-ratio 3.78 -> 1.24). Original note: seamless tiling on every image family (BeginSeamlessTiling on
             DiffusionPipelineBase; ImageFeatures.SeamlessTiling now declared by all 25 image recipes).
             The extension's param gating reads ImageFeatures live from the engine, so the Seamless
             Tileable control stays hidden on non-SDXL families until this pin lands.
             Restore 404s until the engine's alpha.32 publish completes. alpha.32 = Qwen-Image ControlNet (InstantX ControlNet-Union; canny/depth/pose/soft-edge via one union checkpoint). alpha.31 = LoRA on ten more image families (loud zero-match refusal, kohya/bare + ComfyBflDit formats, fused-QKV slice fallback, GGUF merge refusal). alpha.30 = the generic refiner stage (any image model refines any base via pixel hand-off; Refiner.FamilyId carries Swarm's compat-class knowledge). alpha.29 = masked inpaint for HunyuanImage/Lens/Lance + the bespoke HunyuanImage 2.1 VAE encoder (also makes plain img2img real there). alpha.28 = variation seed on every image family (blended in DiffusionPipelineBase.TakeOrCreateNoise at each pipeline's own seed shape) — by design, same as the
             alpha.4/.5/.10/.11/.17 precedent above: merge and publish the engine FIRST, then this pin. -->
        <!-- 2.0.0-alpha.33 = the sampler/scheduler pass (Add schedulers, "more scheduler work", "even more
             sampler work") plus the Wan VAE streamed encode and the Wan-Animate-2 driving-cache reservation
             that makes 480x800x41f fit on a single card. -->
        <!-- 2.0.0-alpha.34 = the sampler selection actually REACHING the pipelines. alpha.33 shipped the seam but
             22 of 26 image recipe pipelines never assigned Scheduler on their inner request, so every flow-match
             family silently ran Euler and the refuse-by-name guards (Wan/LTX/Ideogram 4) never saw the string.
             Also adds SamplingCapabilities (the per-family sampler/schedule table this extension queries in
             ValidateSamplingChoice and serves to the gen-page script) and VideoRequest.Scheduler.
             Restore 404s until the engine's alpha.34 publish completes — by design, same precedent as above:
             merge and publish the engine FIRST, then this pin. -->
        <!-- 2.0.0-alpha.42 = complete latent-preview coverage for every registered image/video family,
             including full temporal video frames for animated WebP previews and direct API/CLI consumers. -->
        <!-- 2.0.0-alpha.44 = side-model download policy change (strict pathing unless user grants consent) + public-side-model
             selection work for LTX-2.5 (upscaler repack selection and no implicit fetch on missing side models). -->
        <!-- 2.0.0-alpha.66 = ImageRequest.ReferenceImages, which the prompt-images mapping above needs: an edit
             model takes extra references as its own conditioning, not through the IP-Adapter slot. Also carries the
             SwarmUI prompt-tag parity work (<weight[N]:>/<alternate:>/<fromto[N]:> now arrive as literal tags) and
             ImageRequest.RemoveBackground — the engine-side cutout, which this extension does NOT set because it
             runs its own Vision pass at the final resolution; setting both would remove the background twice. -->
        <!-- 2.0.0-alpha.75 = the Kohya SD1.5/SDXL LoRA LDM-block-name fix (LoraFormatDetector/KohyaSdMapper) —
             every CivitAI SDXL LoRA using sd-scripts' input_blocks_/output_blocks_/middle_block_ naming was
             previously refused as an undetectable format. -->
        <!-- 2.0.0-alpha.130 = catch-up bump, 2026-09-19 (alpha.75 -> alpha.130). Confirmed via a full Release
             build against this pin that nothing in this extension needed to change: BackendFactory.ValidSelectors/
             Validate/Resolve/Kind/WithOrdinal/ParseOrdinal (everything this file calls) are unchanged; the only
             API surface added in the gap is BackendFactory.ProbeCuda/CudaProbeFailureReason/ResolveProbed (a real
             GPU test-computation, not just device presence) and BackendFactory.CanonicalDeviceKey, both additive.
             ProbeCuda is now used by the new ComputeBackendOptions dropdown below. -->
        <PackageReference Include="HartsyInference" Version="2.0.0-alpha.130" />
    </ItemGroup>

    <!-- LOCAL ENGINE: prebuilt DLLs from the HartsyInference Cli bin (its full runtime closure). Referenced
         directly (Private=true copies them to our output for Swarm's per-extension AssemblyLoadContext),
         plus the CUDA PTX kernels the backend resolves next to the DLL. FIRST build the engine:
         dotnet build src/HartsyInference.Cli/HartsyInference.Cli.csproj -c Release -f net8.0 (in HartsyInference).
         HartsyInference.Engine is the ONE assembly this extension actually programs against (load + generate);
         the rest are its runtime closure, copied so the per-extension ALC can resolve them. -->
    <ItemGroup Condition="'$(UseLocalHartsy)' == 'true'">
        <Reference Include="HartsyInference.Engine"><HintPath>$(HartsyLocalBin)/HartsyInference.Engine.dll</HintPath><Private>true</Private></Reference>
        <Reference Include="HartsyInference.Core"><HintPath>$(HartsyLocalBin)/HartsyInference.Core.dll</HintPath><Private>true</Private></Reference>
        <Reference Include="HartsyInference.ModelAssets"><HintPath>$(HartsyLocalBin)/HartsyInference.ModelAssets.dll</HintPath><Private>true</Private></Reference>
        <Reference Include="HartsyInference.LLM"><HintPath>$(HartsyLocalBin)/HartsyInference.LLM.dll</HintPath><Private>true</Private></Reference>
        <Reference Include="HartsyInference.Diffusion"><HintPath>$(HartsyLocalBin)/HartsyInference.Diffusion.dll</HintPath><Private>true</Private></Reference>
        <Reference Include="HartsyInference.Audio"><HintPath>$(HartsyLocalBin)/HartsyInference.Audio.dll</HintPath><Private>true</Private></Reference>
        <Reference Include="HartsyInference.Video"><HintPath>$(HartsyLocalBin)/HartsyInference.Video.dll</HintPath><Private>true</Private></Reference>
        <Reference Include="HartsyInference.Vision"><HintPath>$(HartsyLocalBin)/HartsyInference.Vision.dll</HintPath><Private>true</Private></Reference>
        <Reference Include="HartsyInference.World"><HintPath>$(HartsyLocalBin)/HartsyInference.World.dll</HintPath><Private>true</Private></Reference>
        <Reference Include="HartsyInference.ThreeD"><HintPath>$(HartsyLocalBin)/HartsyInference.ThreeD.dll</HintPath><Private>true</Private></Reference>
        <Reference Include="HartsyInference.Cpu"><HintPath>$(HartsyLocalBin)/HartsyInference.Cpu.dll</HintPath><Private>true</Private></Reference>
        <Reference Include="HartsyInference.Cuda"><HintPath>$(HartsyLocalBin)/HartsyInference.Cuda.dll</HintPath><Private>true</Private></Reference>
        <Reference Include="HartsyInference.Vulkan"><HintPath>$(HartsyLocalBin)/HartsyInference.Vulkan.dll</HintPath><Private>true</Private></Reference>
        <!-- Engine's private managed deps (normally pulled by the meta-package). -->
        <Reference Include="Google.Protobuf"><HintPath>$(HartsyLocalBin)/Google.Protobuf.dll</HintPath><Private>true</Private></Reference>
        <Reference Include="Microsoft.ML.Tokenizers"><HintPath>$(HartsyLocalBin)/Microsoft.ML.Tokenizers.dll</HintPath><Private>true</Private></Reference>
        <!-- CUDA PTX kernels (the NuGet copies these via its targets; replicate for local). -->
        <Content Include="$(HartsyRepo)/src/HartsyInference.Cuda/Ptx/*.ptx" Link="Ptx/%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest" Condition="Exists('$(HartsyRepo)/src/HartsyInference.Cuda/Ptx')" />
        <Content Include="$(HartsyRepo)/src/HartsyInference.Vulkan/Spv/*.spv" Link="Spirv/%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest" Condition="Exists('$(HartsyRepo)/src/HartsyInference.Vulkan/Spv')" />
    </ItemGroup>

    <!-- Ideogram 4's magic-prompt system prompt, ported verbatim from upstream
         ideogram-oss/ideogram4 (src/ideogram4/magic_prompt_system_prompts/v1.txt). Embedded into the
         extension DLL so the magic-prompt expander has it with no runtime download. Loaded by
         Generation.Ideogram4MagicPrompt via GetManifestResourceNames(). -->
    <ItemGroup>
        <EmbeddedResource Include="Assets/ideogram4_magic_prompt_v1.txt" />
    </ItemGroup>

    <!-- Don't ship duplicates of packages the SwarmUI host already loads. SwarmUI.deps.props makes
         the extension COMPILE against the same SixLabors / Newtonsoft / LiteDB / LLamaSharp / etc.
         the host provides at runtime; ExcludeAssets="runtime" keeps the compile references but stops
         their DLLs from being copied into our output. Without this, CopyLocalLockFileAssemblies above
         would duplicate them and trigger Swarm's "ships X.dll but host already has it" warning (and a
         version-skew hazard). Keep this list in sync with SwarmUI.deps.props. -->
    <ItemGroup>
        <PackageReference Update="FreneticLLC.FreneticUtilities" ExcludeAssets="runtime" />
        <PackageReference Update="Hardware.Info" ExcludeAssets="runtime" />
        <PackageReference Update="LiteDB" ExcludeAssets="runtime" />
        <PackageReference Update="LLamaSharp" ExcludeAssets="runtime" />
        <PackageReference Update="LLamaSharp.Backend.Cpu" ExcludeAssets="runtime" />
        <PackageReference Update="LLamaSharp.Backend.Cuda12" ExcludeAssets="runtime" />
        <PackageReference Update="Newtonsoft.Json" ExcludeAssets="runtime" />
        <PackageReference Update="SixLabors.ImageSharp" ExcludeAssets="runtime" />
        <PackageReference Update="SixLabors.ImageSharp.Drawing" ExcludeAssets="runtime" />
    </ItemGroup>
</Project>