How to use Laya for local game AI in Unity

What Laya is

Laya versus Jev at a glance

What hardware you need

Takeaway: 1 GB of memory, any GPU, a few milliseconds per decision. On a CPU it works for slow decisions only.

How fast is it compared to Jev?

How accurate is it?

Takeaway: Zero-shot, Laya is not Jev. Fine-tuned on your own game it can beat Jev on your own game, in 20 ms with no bill. Plan the fine-tuning step from day one.

How to use Laya in a Unity game

Step 1: run Laya as a local service

pip install "laya[serve]"
LAYA_HOST=127.0.0.1 LAYA_MODELS=english LAYA_DEVICE=cuda LAYA_PRELOAD=1 laya-serve

{
  "state": {
    "time": "early game, about 6 minutes in",
    "our_bases": "one main base, expansion under construction",
    "our_army": "small, mostly infantry, a few tanks. Much smaller than the enemy's",
    "enemy": "scout saw a second enemy base and a barracks with many infantry",
    "resources": "ore is low, energy is fine",
    "last_enemy_action": "two scouts probed our east wall and left"
  },
  "questions": {
    "plan": {
      "type": "choice",
      "instructions": "Pick the best plan for the next minute.",
      "criteria": {
        "defend_and_expand": "finish the expansion and keep the army home",
        "tech_up": "build the factory upgrade before more units",
        "build_army": "spend everything on more combat units",
        "attack_now": "send the current army at the nearest enemy base",
        "harass_economy": "send a small fast group to kill enemy harvesters"
      }
    },
    "threat": {
      "type": "score",
      "instructions": "How much danger is our main base in right now?",
      "criteria": ["safe", "being watched", "attack likely soon", "under attack"]
    },
    "enemy_rushing": {
      "type": "noul",
      "instructions": "The enemy is preparing an early rush.",
      "criteria": {
        "true": "the enemy is massing cheap units for an early attack",
        "false": "the enemy is playing a normal, slower build"
      }
    }
  }
}

{
  "answers": {
    "plan": {
      "type": "choice",
      "choice": "defend_and_expand",
      "probabilities": { "defend_and_expand": 0.47, "tech_up": 0.09, "build_army": 0.31, "attack_now": 0.04, "harass_economy": 0.09 },
      "confidence": 0.38,
      "answer_confidence": 0.47
    },
    "threat": { "type": "score", "score": 1.6, "probabilities": { "0": 0.10, "1": 0.35, "2": 0.40, "3": 0.15 }, "confidence": 0.29 },
    "enemy_rushing": { "type": "noul", "noul": 0.62 }
  },
  "routing": { "model": "english" },
  "usage": { "input_tokens": 231, "output_tokens": 0 }
}

Step 2: typed questions in C#

using System.Collections.Generic;

public static class LayaQ
{
    public static Dictionary<string, object> Choice(string instructions, IReadOnlyDictionary<string, string> options)
        => new Dictionary<string, object>
        {
            ["type"] = "choice",
            ["instructions"] = instructions,
            ["criteria"] = options
        };

    public static Dictionary<string, object> Score(string instructions, params string[] levelsLowToHigh)
        => new Dictionary<string, object>
        {
            ["type"] = "score",
            ["instructions"] = instructions,
            ["criteria"] = levelsLowToHigh
        };

    public static Dictionary<string, object> Noul(string statement, string ifTrue, string ifFalse)
        => new Dictionary<string, object>
        {
            ["type"] = "noul",
            ["instructions"] = statement,
            ["criteria"] = new Dictionary<string, string> { ["true"] = ifTrue, ["false"] = ifFalse }
        };
}

Step 3: the client

using System;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Collections.Generic;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

public sealed class LayaClient
{
    private static readonly HttpClient Http = new HttpClient();
    private readonly string _endpoint;

    // Same wire format as TypeSafe's Jev. If a server build ever wants a hosted model instead,
    // this URL is the only thing that changes.
    public LayaClient(string endpoint = "http://127.0.0.1:8000/v1/systemone")
    {
        _endpoint = endpoint;
    }

    // Returns the "answers" object, or null if the sidecar did not answer in time.
    public async Task<JObject> AskAsync(object state, Dictionary<string, object> questions, int timeoutMs = 250)
    {
        var body = new JObject
        {
            ["state"] = JToken.FromObject(state),
            ["questions"] = JToken.FromObject(questions)
        };

        using var cts = new CancellationTokenSource(timeoutMs);
        using var content = new StringContent(body.ToString(Formatting.None), Encoding.UTF8, "application/json");

        try
        {
            var response = await Http.PostAsync(_endpoint, content, cts.Token);
            if (!response.IsSuccessStatusCode) return null;

            var json = JObject.Parse(await response.Content.ReadAsStringAsync());
            return json["answers"] as JObject;
        }
        catch (Exception)
        {
            return null; // timeout, sidecar not up yet, or a bad response: the caller falls back
        }
    }
}

public static class LayaAnswers
{
    public static string Choice(JObject answers, string id) => (string)answers[id]?["choice"];
    public static float AnswerConfidence(JObject answers, string id) => (float?)answers[id]?["answer_confidence"] ?? 0f;
    public static float Score(JObject answers, string id) => (float?)answers[id]?["score"] ?? 0f;
    public static float Noul(JObject answers, string id) => (float?)answers[id]?["noul"] ?? 0f;
}

Step 4: starting and stopping the sidecar from Unity

using System.Diagnostics;
using System.Net.Http;
using System.Threading.Tasks;
using UnityEngine;

public sealed class LayaSidecar : MonoBehaviour
{
    // During development this is "laya-serve" from your Python environment.
    // For shipping, point it at the ggmlc "laya" binary in StreamingAssets and pass
    // "serve laya_english_q8_0.gguf --port 8000" as arguments (and adapt the client route).
    [SerializeField] private string executable = "laya-serve";
    [SerializeField] private string arguments = "";
    [SerializeField] private int threadsIfCpu = 4;   // physical cores, not logical. See the hardware section

    public bool Ready { get; private set; }

    private Process _process;

    private async void Start()
    {
        var psi = new ProcessStartInfo
        {
            FileName = executable,
            Arguments = arguments,
            UseShellExecute = false,
            CreateNoWindow = true,
        };
        psi.Environment["LAYA_HOST"] = "127.0.0.1";
        psi.Environment["LAYA_MODELS"] = "english";
        psi.Environment["LAYA_PRELOAD"] = "1";
        psi.Environment["LAYA_THREADS"] = threadsIfCpu.ToString();

        _process = Process.Start(psi);

        // Cold start is a few seconds on a GPU and longer on a CPU. Poll health, do not guess.
        using var http = new HttpClient();
        for (int i = 0; i < 120 && !Ready; i++)
        {
            try
            {
                var r = await http.GetAsync("http://127.0.0.1:8000/health");
                Ready = r.IsSuccessStatusCode;
            }
            catch { /* not listening yet */ }

            if (!Ready) await Task.Delay(500);
        }
    }

    private void OnApplicationQuit()
    {
        if (_process != null && !_process.HasExited) _process.Kill();
    }
}

Step 5: the commander

using System.Collections;
using System.Collections.Generic;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;
using UnityEngine;

public sealed class AICommander : MonoBehaviour
{
    [SerializeField] private LayaSidecar sidecar;
    [SerializeField] private float intervalSeconds = 12f;

    // Measure this after fitting temperatures on your own data. The shipped checkpoints are
    // overconfident, so do not copy a number from a blog post, including this one.
    [SerializeField, Range(0f, 1f)] private float confidenceThreshold = 0.5f;

    private LayaClient _laya;
    private ScriptedCommander _fallback;   // the AI you already have
    private BuildOrders _buildOrders;
    private BaseDefense _defense;
    private MatchReader _match;            // whatever reads your match state

    private void Awake()
    {
        _laya = new LayaClient();
    }

    private void Start()
    {
        StartCoroutine(Loop());
    }

    private IEnumerator Loop()
    {
        var wait = new WaitForSeconds(intervalSeconds);
        while (true)
        {
            yield return wait;
            if (sidecar.Ready) _ = ThinkAsync();
            else _fallback.Tick();
        }
    }

    private async Task ThinkAsync()
    {
        // 1. Words, not numbers. The game does the math and writes the sentence.
        var state = new Dictionary<string, string>
        {
            ["time"] = _match.PhaseInWords(),                    // "early game, about 6 minutes in"
            ["our_bases"] = _match.BasesInWords(),               // "one main base, expansion under construction"
            ["our_army"] = _match.ArmyComparedToEnemyInWords(),  // "much smaller than the enemy's, mostly infantry"
            ["enemy"] = _match.LatestScoutReportInWords(),       // "a second enemy base and a barracks with many infantry"
            ["resources"] = _match.ResourcesInWords(),           // "ore is low, energy is fine"
            ["last_enemy_action"] = _match.LastEnemyActionInWords()
        };

        // 2. Only offer plans the AI can actually execute right now.
        var legal = new Dictionary<string, string>();
        if (_match.CanAffordExpansion)  legal["defend_and_expand"] = "finish the expansion and keep the army home";
        if (_match.FactoryUpgradeReady) legal["tech_up"] = "build the factory upgrade before more units";
        legal["build_army"] = "spend everything on more combat units";
        if (_match.HasArmy)             legal["attack_now"] = "send the current army at the nearest enemy base";
        if (_match.HasFastUnits)        legal["harass_economy"] = "send a small fast group to kill enemy harvesters";

        var questions = new Dictionary<string, object>
        {
            ["plan"] = LayaQ.Choice("Pick the best plan for the next minute.", legal),
            ["threat"] = LayaQ.Score("How much danger is our main base in right now?",
                                     "safe", "being watched", "attack likely soon", "under attack"),
            ["enemy_rushing"] = LayaQ.Noul("The enemy is preparing an early rush.",
                                           "the enemy is massing cheap units for an early attack",
                                           "the enemy is playing a normal, slower build")
        };

        // 3. Ask. The previous plan keeps running while this is in flight.
        JObject answers = await _laya.AskAsync(state, questions);

        // 4. Validate, then fall back to the scripted commander if anything looks off.
        string plan = answers != null ? LayaAnswers.Choice(answers, "plan") : null;
        if (plan == null || !legal.ContainsKey(plan) || LayaAnswers.AnswerConfidence(answers, "plan") < confidenceThreshold)
        {
            _fallback.Tick();
            return;
        }

        // 5. Hand the plan to the systems that already know how to do it.
        _buildOrders.Switch(plan);
        _defense.SetAlertLevel(LayaAnswers.Score(answers, "threat"));
        if (LayaAnswers.Noul(answers, "enemy_rushing") > 0.7f) _buildOrders.QueueEmergencyDefenses();
    }
}

Step 6: bases and squads in the same call

private Dictionary<string, object> OperationsQuestions(IEnumerable<PlayerBase> bases, IEnumerable<Squad> squads)
{
    var questions = new Dictionary<string, object>();

    foreach (var b in bases)
    {
        questions[$"threat_{b.Id}"] = LayaQ.Score(
            $"How much danger is the {b.NameInWords} in right now?",
            "safe", "being watched", "attack likely soon", "under attack");
    }

    foreach (var s in squads)
    {
        // Legal orders only. If there is nowhere to retreat to, pull_back is not on the list.
        var legal = new Dictionary<string, string>();
        if (s.HasVisibleEnemy)  legal["engage"] = "attack the enemy group in front of us";
        legal["hold"] = "hold this position";
        if (s.HasRetreatPoint)  legal["pull_back"] = "fall back to the nearest friendly base";
        if (s.HasFastUnits)     legal["flank"] = "circle around and hit the enemy from the side";

        questions[$"order_{s.Id}"] = LayaQ.Choice($"Pick the best order for the {s.NameInWords}.", legal);
    }

    return questions;
}

Step 7: recording training data from Unity

using System.Collections.Generic;
using System.IO;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

public sealed class DecisionRecorder : System.IDisposable
{
    private readonly StreamWriter _writer;

    public DecisionRecorder(string path)
    {
        _writer = new StreamWriter(path, append: true);
    }

    public void Record(Dictionary<string, string> state, Dictionary<string, string> legalPlans, string planThePlayerChose)
    {
        var row = new JObject
        {
            ["state"] = JToken.FromObject(state),
            ["questions"] = JToken.FromObject(new Dictionary<string, object>
            {
                ["plan"] = LayaQ.Choice("Pick the best plan for the next minute.", legalPlans)
            }),
            ["labels"] = new JObject { ["plan"] = planThePlayerChose }
        };
        _writer.WriteLine(row.ToString(Formatting.None));
    }

    public void Dispose() => _writer.Dispose();
}

Running Laya inside the Unity process

Multiplayer and replays

Takeaway: Laya as a local service, everything else in C#. Batch decisions into one call, validate against the legal list, fall back to your scripted AI, and record the same state sentences during playtests for fine-tuning.

Other local options

Is it worth looking into?