From eef15fe7a6379e4065c70f941db3adc67ec895e3 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Mon, 19 Oct 2026 19:48:22 -0500 Subject: [PATCH 01/42] different strategy for console exit hang the original approach, while conceptually cleaner, does not work in Wine/Proton because Wine's `WaitForMultipleObjects` is a pile of dog poop that almost never works this approach should address both the `FreeConsole` hangs observed with newer versions of WIndows 10 and 11 as well as avoiding Wine's deficiencies --- library/Console-windows.cpp | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/library/Console-windows.cpp b/library/Console-windows.cpp index bb6b1fc8fb..11648f90f6 100644 --- a/library/Console-windows.cpp +++ b/library/Console-windows.cpp @@ -46,6 +46,7 @@ POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include #include #include #include @@ -73,7 +74,7 @@ namespace DFHack rawmode = 0; console_in = 0; console_out = 0; - exit_event = 0; + exiting = false; ConsoleWindow = 0; default_attributes = 0; state = con_unclaimed; @@ -282,21 +283,17 @@ namespace DFHack inf.dwCursorPosition.X = (SHORT)plen; SetConsoleCursorPosition(console_out, inf.dwCursorPosition); - // exit_event comes first so shutdown wins if input is also - // pending. The console input handle is signaled whenever its - // input queue is non-empty, so the read below cannot block. - const HANDLE wait_handles[] = { exit_event, console_in }; while (1) { INPUT_RECORD rec; DWORD count; lock->unlock(); - DWORD wait_result = WaitForMultipleObjects(2, wait_handles, FALSE, INFINITE); + // shutdown() queues a dummy input record to wake this read; + // the exiting flag distinguishes the wake from real input. + BOOL success = ReadConsoleInputA(console_in, &rec, 1, &count); lock->lock(); - if (wait_result != WAIT_OBJECT_0 + 1 - || ReadConsoleInputA(console_in, &rec, 1, &count) == 0) { + if (exiting || !success) return Console::SHUTDOWN; - } if (rec.EventType != KEY_EVENT || !rec.Event.KeyEvent.bKeyDown) continue; switch (rec.Event.KeyEvent.wVirtualKeyCode) @@ -402,7 +399,7 @@ namespace DFHack int rawmode; HANDLE console_in; HANDLE console_out; - HANDLE exit_event; + std::atomic exiting; HWND ConsoleWindow; HWND MainWindow; WORD default_attributes; @@ -486,9 +483,6 @@ bool Console::init(bool) d->console_owned = AllocConsole() != FALSE; d->ConsoleWindow = GetConsoleWindow(); wlock = new std::recursive_mutex(); - d->exit_event = CreateEventW(NULL, TRUE, FALSE, NULL); - if (!d->exit_event) - return false; if (d->console_owned) { HMENU hm = GetSystemMenu(d->ConsoleWindow,false); @@ -546,11 +540,15 @@ bool Console::init(bool) bool Console::shutdown(void) { assert(inited); + d->exiting = true; std::lock_guard lock{*wlock}; - // Wake any thread waiting on console input so it leaves prompt_loop - // before we detach or reconfigure the console below. - if (d->exit_event) - SetEvent(d->exit_event); + // Wake any thread blocked in ReadConsoleInputA by queueing a dummy + // input record, so it is no longer inside a console call and can + // observe the exiting flag before we detach the console below. + INPUT_RECORD rec = { 0 }; + rec.EventType = FOCUS_EVENT; + DWORD written; + WriteConsoleInputA(d->console_in, &rec, 1, &written); if (d->console_owned) { FreeConsole(); From 438faf3b87ff3a308c0a8501e91228a10cdaeec4 Mon Sep 17 00:00:00 2001 From: DFHack-Urist via GitHub Actions <63161697+DFHack-Urist@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:34:38 +0000 Subject: [PATCH 02/42] Auto-update submodules scripts: master --- scripts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts b/scripts index e041f5b2af..4c4f5bf510 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit e041f5b2af8400e2a302f12144dbd494a3ec5e09 +Subproject commit 4c4f5bf510e22fb96857328638e93ded3f608d0e From f10ac67c4ad13efb38fa1d7c2627bb091cf22a53 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Tue, 20 Oct 2026 09:21:13 -0500 Subject: [PATCH 03/42] add g_src analysis notes --- docs/dev/g_src-notes.rst | 804 +++++++++++++++++++++++++++++++++++++++ docs/dev/index.rst | 1 + 2 files changed, 805 insertions(+) create mode 100644 docs/dev/g_src-notes.rst diff --git a/docs/dev/g_src-notes.rst b/docs/dev/g_src-notes.rst new file mode 100644 index 0000000000..e7c1a5e221 --- /dev/null +++ b/docs/dev/g_src-notes.rst @@ -0,0 +1,804 @@ +.. _g_src-notes: + +######################################### +Notes on the Bay12 ``g_src`` sources +######################################### + +This document describes the open-source portion of Dwarf Fortress that Bay12 +ships in its Linux packages (``g_src``), and how that code maps onto the +symbols and types defined in `df-structures`_ (the ``df.g_src.*.xml`` files and +related globals such as ``enabler``, ``gps``, ``gview``, ``init``, +``musicsound``). + +.. _df-structures: https://github.com/DFHack/df-structures + +Bay12 releases this code so that Linux packagers can rebuild the parts of DF +that link against distribution-specific libraries (SDL2, OpenGL, curses, etc.). + +The reference tree used here is the ``g_src`` folder from the Linux Classic build of DF 53.16 +as available for `download from Bay12 `__. Line numbers are approximate. + +.. contents:: Contents + :local: + +Source inventory and XML correspondence +======================================= + +.. list-table:: + :header-rows: 1 + + * - g_src file + - Role + - df-structures file + * - ``enabler.h``, ``enabler.cpp`` + - ``enablerst`` — SDL main loop, frame scheduler, thread coordination, + ``renderer`` base class, ``textures``, ``Either``, ``zoom_commands``, + ``texture_fullid`` + - ``df.g_src.enabler.xml`` + * - ``enabler_input.h``, ``enabler_input.cpp`` + - ``enabler_inputst`` (base of ``enablerst``), key translation, repeat + timing, macros + - ``df.g_src.enabler.xml`` (parts), ``df.g_src.keybindings.xml`` + * - ``graphics.h``, ``graphics.cpp`` + - ``graphicst`` (``gps``), ``graphic_viewportst``, + ``graphic_map_portst``, ``interface_setst``, ``texblitst``, + screen/tile buffers + - ``df.g_src.graphics.xml`` + * - ``renderer_2d.hpp`` + - ``renderer_2d_base``, ``renderer_2d`` — SDL_Renderer based tile + renderer, texture cache + - ``df.g_src.renderer_2d.xml`` + * - ``renderer_curses.cpp`` + - ``renderer_curses`` (PRINT_MODE:TEXT) + - ``df.g_src.renderer_2d.xml`` + * - ``renderer_offscreen.cpp`` + - offscreen renderer for benchmarking + - ``df.g_src.renderer_2d.xml`` + * - ``renderer_opengl.hpp`` + - legacy OpenGL renderer (not compiled — ``#include`` commented out in + ``enabler.cpp:194``) + - n/a + * - ``interface.h``, ``interface.cpp`` + - ``interfacest`` (``gview``), scrolling/string-entry helpers + - ``df.g_src.interface.xml``, ``df.interface.xml`` + * - ``ViewBase.h``, ``ViewBase.cpp`` + - ``viewscreenst``, ``abstract_interfacest``, ``widgets::*`` widget tree + - ``df.g_src.ViewBase.xml`` + * - ``keybindings.h``, ``keybindings.cpp`` + - ``InterfaceKey`` enum, key binding names bimaps + - ``df.g_src.keybindings.xml`` + * - ``KeybindingScreen.*`` + - keybinding-editing UI (viewscreen) + - ``df.g_src.KeybindingScreen.xml`` + * - ``init.h``, ``init.cpp`` + - ``initst`` (``init``) — init.txt/colors.txt parsing, font texpos + tables + - ``df.g_src.init.xml`` + * - ``files.h``, ``files.cpp``, ``find_files.h`` + - ``filest``, ``stringvectst``, path handling, compressed file I/O + - ``df.g_src.files.xml``, ``df.g_src.basics.xml`` + * - ``basics.h``, ``basics.cpp`` + - misc helpers, ``gamelog``/``errorlog``, ``flagarrayst`` + - ``df.g_src.basics.xml`` + * - ``texture_handler.h``, ``textures.cpp`` + - ``texture_handlerst``, ``palettest``, tile-page loading + - ``df.g_src.texture_handler.xml`` + * - ``platform.h``, ``win32_compat.cpp`` + - Win32 API shims for Linux (``GetTickCount``, ``MessageBox``, + ``DWORD``, ...) + - ``df.g_src.platform.xml`` + * - ``mail.hpp`` + - ``Chan`` channel, ``Either`` + - modeled inline in ``df.g_src.enabler.xml`` + * - ``music_and_sound_g.h``, ``music_and_sound*.cpp`` + - ``musicsoundst`` (``musicsound``), ``Song``/``MusicEvent``/ + ``MusicContext`` enums + - ``df.g_src.music_and_sound_g.xml`` + * - ``random.h``, ``random.cpp`` + - Mersenne Twister wrapper, ``trandom*``, seed stack + - ``df.random_objects.xml`` + * - ``command_line.h``, ``command_line.cpp`` + - ``command_linest`` + - ``df.g_src.command_line.xml`` + * - ``lua.h``, ``lua.cpp`` + - ``LuaState`` RAII wrapper, DF Lua globals + - not mapped (see "Other modules" below) + * - ``textlines.h`` + - ``textlinesst`` raw-file line loader + - ``df.g_src.basics.xml`` + * - ``svector.h``, ``bimap.h``, ``pool.h``, ``array_vec.hpp``, + ``template.h``, ``endian.h``, ``resize++.h`` + - internal utility containers/algorithms + - mostly not mapped (STL-internal) + * - ``dfhooks.h``, ``dfhooks.cpp`` + - the DF ↔ DFHack plugin boundary + - n/a — see "DFHack hook placement" below + +Process architecture — the two threads +====================================== + +``enabler.cpp`` implements a two-thread design. Understanding which thread +runs what is the single most important thing about ``g_src`` for DFHack. + +Boot sequence (``main``, ``enabler.cpp:1270``) +---------------------------------------------- + +1. ``SDL_Init(SDL_INIT_TIMER)`` on the **main thread**. +2. ``enabler.renderer_threadid = SDL_ThreadID()`` — the **main thread is the + renderer/event thread** (``enabler.cpp:1282``). +3. ``SDL_CreateThread(call_loop, ...)`` spawns the **simulation thread**, + which runs ``enablerst::async_loop()`` (``enabler.cpp:1265-1268``, + ``746``). +4. ``init.begin()`` parses ``init_default.txt``, ``prefs/init.txt``, + ``colors.txt`` into ``init``/``gps`` (``init.cpp:93``). +5. ``SDL_InitSubSystem(SDL_INIT_VIDEO)``, ``keybinding_init()``, keybindings + loaded (``prefs/interface.txt`` then ``data/init/interface.txt`` — prefs + first because ``load_keybindings`` does not overwrite). +6. ``enabler.loop(cmdline)`` → ``beginroutine()`` (closed source) → renderer + creation (``renderer_2d`` unless PRINT_MODE:TEXT → ``renderer_curses``) → + ``hooks_init()`` → ``eventLoop_SDL()`` (``enabler.cpp:1073-1125``). + +So: the thread that runs ``main()`` owns SDL events, the SDL window/renderer, +and presentation. The created thread owns the simulation and the +screen-buffer writes. + +The channels (``mail.hpp``, ``enabler.h:891-899``) +-------------------------------------------------- + +Three ``Chan`` queues plus one semaphore, all members of ``enablerst``: + +- ``async_tobox : Chan`` — **main → sim** commands: ``pause``, + ``start``, ``render``, ``inc`` (grant N sim frames), ``set_fps``. +- ``async_frombox : Chan`` — **sim → main** messages: ``quit``, + ``complete``, ``set_fps``, ``set_gfps``, ``push_resize``, ``pop_resize``, + ``reset_textures``, ``show_message``. +- ``async_zoom : Chan`` — **sim → main** zoom/fullscreen + requests (``zoom_display()`` writes here; the event loop drains it). +- ``async_fromcomplete : std::binary_semaphore`` — released by the main + thread after it has serviced an ``async_frombox`` request; callers block + on it (see ``override_grid_size``, ``set_fps``, ``set_gfps``, + ``show_message_box``). + +``async_msg`` uses a ``union`` for payloads: ``fps``, ``{x,y}``, or +``{text, caption, type}`` — the wide-string pointers for ``show_message`` +are raw borrowed pointers whose lifetime is guarded by the caller blocking +on ``async_fromcomplete`` until the main thread has shown the box +(``enabler.cpp:733-738``; ``enabler.h`` ``show_message_box``). + +The sim thread — ``enablerst::async_loop`` (``enabler.cpp:746``) +---------------------------------------------------------------- + +The sim thread spins: + +1. Drain ``async_tobox`` (blocking ``read`` when paused or out of frame + budget, ``try_read`` otherwise): + + - ``pause`` → set ``async_paused``, ack with ``async_msg::complete``. + - ``start`` → unpause, reset ``async_frames``. + - ``render`` → if ``ENABLERFLAG_RENDER``: ``renderer->swap_arrays()``, + ``render_things()`` (fills ``gps`` buffers), clear flag, + ``update_gfps()``; then ack ``complete``. **This is the screen-render + tick, on the sim thread.** + - ``inc`` → add to ``async_frames`` budget (clamped to ``fps*3``). + - ``set_fps`` → update local fps. + +2. If unpaused and budget remains (or ``ENABLERFLAG_MAXFPS``): call + ``mainloop()`` — the closed-source simulation + UI step — then + ``simticks++``, ``clear_text_input()``. +3. ``hooks_update()`` **every iteration**, on the sim thread + (``enabler.cpp:812``). This is where ``DFHack::Core::Update()`` runs. + +If ``mainloop()`` returns true, it posts ``async_msg::quit`` and exits; the +main thread sees ``quit`` in ``async_wait``, sets ``loopvar=0``, and both +loops end. + +The main thread — ``eventLoop_SDL`` + ``do_frame`` (``enabler.cpp:892``, ``816``) +--------------------------------------------------------------------------------- + +Each iteration: + +1. Drain ``async_zoom``; if any zooms arrive, **pause the sim first** + (``pause_async_loop`` writes ``pause`` to ``async_tobox`` and blocks in + ``async_wait`` until ``complete``). +2. ``SDL_PollEvent`` loop — again, the first event triggers + ``pause_async_loop``, so **all input processing happens while the sim + thread is parked**. For each event: ``hooks_sdl_event(&event)`` (main + thread, can consume the event entirely), then the switch: keys/wheel/ + quit → ``enabler.add_input`` (input state machine); buttons → + ``enabler.mouse_*``; window events → resize/cursor-focus/ + ``force_full_display_count``/``ENABLERFLAG_RENDER``. +3. Mouse state → ``gps.precise_mouse_x/y``, ``gps.mouse_x/y``, + ``enabler.tracking_on`` (written while sim is paused). +4. ``unpause_async_loop()``. +5. ``hooks_sdl_loop_fn()`` — main thread, just before ``do_frame``. +6. ``do_frame()``: + + - accumulate wall-clock time into ``outstanding_frames`` (sim) and + ``outstanding_gframes`` (graphics), capped at 3 gframes; + - ``inc`` command grants that many sim ticks; + - each gframe: send ``render``, ``async_wait()`` for ``complete`` + (servicing ``set_fps``/``set_gfps``/``push_resize``/``pop_resize``/ + ``reset_textures``/``show_message`` while it waits), then + ``gps.main_thread_requesting_reshape`` handling, + ``tidy_tile_cache()``, ``display()``, ``render()`` → + ``SDL_RenderPresent``. + - ``emit_logs()`` when it sleeps. + +7. ``musicsound.update()``. + +What the pause protocol actually protects +----------------------------------------- + +``pause_async_loop`` only stops ``mainloop()`` and ``render_things`` — it +does **not** lock ``gps``. The buffers are protected by *protocol*, not +mutexes: + +- ``gps.screen``/``screentexpos*`` (the "front" buffers) are written by the + sim thread inside the ``render`` command handler, *before* it posts + ``complete``. +- ``renderer::screen``/``screentexpos*`` (the "back" buffers) are read by + the main thread in ``display()``, *after* ``complete`` arrives. +- ``renderer::swap_arrays()`` (``enabler.cpp:513``) exchanges the + front/back pointer sets, so the sim thread always draws into the buffer + the renderer is *not* currently scanning. The swap runs on the sim + thread, inside the ``render`` command, and completes before ``complete`` + is acked — so there is a happens-before edge via the channel. +- The per-viewport ``screentexpos_*_old`` arrays are likewise rotated in + ``swap_arrays`` so ``display()`` can diff them against the fresh arrays + for incremental redraws (``enabler.cpp:264-398``). + +Things that are **not** safe by protocol (see "Problematic code" below): +``enabler.flag`` (``ENABLERFLAG_RENDER`` written by main, read by sim), +``enabler.async_paused`` (written by ``set_fps`` on whichever thread calls +it), ``simticks``/``gputicks``/``async_frames`` (plain ints read +cross-thread), ``gps.mouse_x``/``mouse_y`` (written by main while +"paused", read by sim at arbitrary times from viewscreen code — races are +possible because the sim thread resumes after ``unpause_async_loop``), and +input statics in ``enabler_input.cpp`` (``timeline``, ``pressed_keys``, +``modState``). + +DFHack hook placement (``dfhooks.cpp``) +--------------------------------------- + +``g_src/dfhooks.cpp`` is a tiny trampoline: ``hooks_init()`` ``dlopen``\ s +``./libdfhooks.so`` (or ``dfhooks.dll``/``libdfhooks.dylib``) and resolves +``dfhooks_init``, ``dfhooks_shutdown``, ``dfhooks_update``, +``dfhooks_prerender``, ``dfhooks_sdl_event``, ``dfhooks_ncurses_key``, +``dfhooks_sdl_loop`` by name. + +``libdfhooks`` itself is the **chainloader** (built from +``depends/dfhooks`` in the DFHack repo): it discovers ``dfhooks_*``/ +``libdfhooks_*.so`` libraries, reads each one's optional +``dfhooks_priority`` int32 (lower runs earlier; DFHack exports 100), calls +each library's optional ``dfhooks_preinit`` at load time, and fans the +other callbacks out in priority order (shutdown in reverse). Note: despite +DFHack's comment in ``library/Hooks.cpp``, ``dfhooks_init``/ +``dfhooks_preinit`` run *after* the sim thread is spawned — +``SDL_CreateThread`` precedes ``enabler.loop`` → ``hooks_init`` in +``main`` (``enabler.cpp:1289``, ``1119``). + +.. list-table:: + :header-rows: 1 + + * - g_src call site + - Exported symbol + - Runs on + - DFHack endpoint (``library/Hooks.cpp``) + * - ``enabler.loop``, after renderer creation, before event loop + - ``dfhooks_init`` + - main + - ``Core::InitMainThread`` + console/command-line checks + * - after ``eventLoop_SDL`` exits + - ``dfhooks_shutdown`` + - main + - ``Core::Shutdown`` + * - every ``async_loop`` iteration + - ``dfhooks_update`` + - **sim** + - ``Core::Update`` (plugin ``onupdate``, ``Gui::*``, etc.) + * - ``render_things``, before top viewscreen ``render()`` + - ``dfhooks_prerender`` + - **sim** + - (currently a stub) + * - every SDL event + - ``dfhooks_sdl_event`` + - **main** + - ``Core::DFH_SDL_Event`` → SDL events for tools + * - every event-loop iteration, before ``do_frame`` + - ``dfhooks_sdl_loop`` + - **main** + - ``Core::DFH_SDL_Loop`` (mouse/display side work) + * - every ncurses key (PRINT_MODE:TEXT only) + - ``dfhooks_ncurses_key`` + - main + - ``Core::DFH_ncurses_key`` + +Practical consequences: + +- Code in ``dfhooks_update`` runs on the sim thread, concurrent with the + main thread's ``display()`` — it must not touch ``renderer.*`` buffers + or SDL. ``CoreSuspender`` exists because even *this* does not stop the + main thread from scanning buffers; it suspends via the DFHack-managed + mutex. +- ``dfhooks_sdl_event``/``dfhooks_sdl_loop`` run while the sim thread is + paused **only for the duration of the event drain** — ``unpause`` + happens before ``hooks_sdl_loop_fn``/``do_frame``, so + ``dfhooks_sdl_loop`` runs concurrent with the sim thread. It is safe for + SDL calls and for reading ``gps`` display-side state, not for mutating + simulation data. +- A ``dfhooks_sdl_event`` handler that consumes an event stops DF's own + processing of it entirely (the ``continue`` at ``enabler.cpp:932``). + +``enablerst`` structure notes (``enabler.h:840``, ``df.g_src.enabler.xml``) +=========================================================================== + +``enablerst`` publicly inherits ``enabler_inputst`` — so +``enabler.add_input``, ``enabler.get_input``, keybinding and macro methods +all live on ``enabler``. + +Field-by-field: + +- ``renderer`` (``renderer*``) — the active renderer; ``renderer_2d`` in + normal SDL builds. Virtual interface in ``enabler.h:733-852``; XML + models all vmethods (``df.g_src.enabler.xml`` ````). +- Timing: ``fps``/``gfps`` are **floats** (target rates), + ``outstanding_frames``/``outstanding_gframes`` are float accumulators + drained by ``do_frame``. ``frame_timings``/``gframe_timings`` + sums + implement the 50-frame rolling average behind + ``calculate_fps``/``calculate_gfps`` (``enabler.cpp:1185-1213``). +- ``simticks``, ``gputicks`` — plain ``int``, incremented on sim and main + threads respectively; also read by the *other* thread (e.g. + ``Event::tick`` in input code) → benign-looking but technically racy. +- ``flag`` (``ENABLERFLAG_*``): ``RENDER`` = "a gframe is due; sim should + run ``render_things``"; ``MAXFPS`` = uncapped sim; ``BASIC_TEXT`` = use + the basic (non-antialiased) font texposes. +- Input: ``mouse_lbut``/``mouse_rbut``/``mouse_mbut`` (+``_down``/ + ``_lift`` edge flags), ``tracking_on`` (mouse inside a valid grid + cell), ``listening_to_text``/``last_text_input`` (32-byte UTF-8 + text-input buffer, filled by ``SDL_TEXTINPUT`` on the main thread, + cleared by sim thread each tick, read via ``get_text_input()`` by + viewscreens that want text). +- ``clock`` — approx. ``SDL_GetTicks`` refreshed per ``do_frame``; cheap + tick for GC-ish logic. +- ``command_line`` — the reconstructed command line string; the actual + ``command_linest`` is ``game->command_line`` (``df.game_g.xml`` / + ``df.g_src.command_line.xml``). +- ``fullscreen_state`` (``FullscreenState`` bitfield), + ``overridden_grid_sizes`` (deque used by + ``override_grid_size``/``release_grid_size``; note the FIXME at + ``enabler.cpp:1157`` about unmatched release calls). +- ``last_message_result`` — ``std::atomic_int``; result of the last + ``MessageBox`` call, set on the main thread in + ``win32_compat.cpp:142``. +- ``must_do_render_things_before_display`` — when a viewport reshape is + pending (``gps.main_thread_requesting_reshape``), ``render_things`` + leaves this true so ``do_frame`` performs the reshape on the main + thread before the next ``display()`` (``enabler.cpp:849-858``, + ``1253``). +- ``textures`` (``enabler_textures``/``textures`` class, + ``enabler.h:557``) — raw ``SDL_Surface`` catalog backing ``texpos`` + values: ``raws`` vector, ``free_spaces``, ``init_texture_size``, + ``uploaded`` flag. ``add_texture``, ``clone_texture``, + ``grayscale_texture``, ``delete_all_post_init_textures`` etc. in + ``textures.cpp``. +- Methods of interest: ``set_fps``/``set_gfps`` (thread-agnostic — they + forward via ``async_frombox`` when called off the renderer thread), + ``override_grid_size``/``release_grid_size``, ``zoom_display``, + ``show_message_box``, ``clean_cached_tile``, + ``get_current_interface_tile_dims``, ``main_renderer()``. + +``graphicst`` and the tile pipeline (``graphics.h:5436``, ``df.g_src.graphics.xml``) +==================================================================================== + +``gps`` is the screen-state singleton. Everything a viewscreen draws lands +in its arrays. + +The screen cell +--------------- + +- Indexing is **column-major**: cell ``(x, y)`` is at ``x*dimy + y`` + (confirmed throughout, e.g. ``addchar``, ``enabler.cpp:38``). +- ``screen`` is ``dimx*dimy*8`` bytes; 8 bytes per cell: ``s[0]`` = + character byte; ``s[1..3]`` = foreground RGB; ``s[4..6]`` = background + RGB; ``s[7]`` = type byte (``GRAPHICSTYPE_TTF``/``_TTFCONT`` — + 0xff/0xfe, currently vestigial since TTF paths are commented out). +- ``screenf``/``screenb``/``screenbright`` + ``use_old_16_colors`` are the + "pen". ``changecolor(f,b,br)`` selects the 16-color palette + (``uccolor[16][3]``); ``color3ub``/``bcolor3ub`` set true RGB + (``screen_color_*`` fields). Both are written into the cell's RGB bytes. +- ``locate(y, x)`` sets the cursor ``screenx``/``screeny`` (note **y + first**); ``addchar`` writes a cell then ``screenx += advance``. + Clipping via ``setclipping``/``clipx``/``clipy``. + +The texpos layer set +-------------------- + +For each cell there is a parallel set of ``long`` texpos arrays: + +- ``screentexpos`` — the base graphical tile (overrides the char when + ``INIT_DISPLAY_FLAG_USE_GRAPHICS``). +- ``screentexpos_lower`` — underlay (beneath text/tiles). +- ``screentexpos_anchored`` + ``_x``/``_y`` — multi-cell anchored tiles + (creatures/buildings bigger than one tile). Subordinate cells get + ``SCREENTEXPOS_FLAG_ANCHOR_SUBORDINATE`` plus packed 6-bit offsets + (``ANCHOR_X_COORD>>6``, ``ANCHOR_Y_COORD>>12``, 64x64 limit, + ``graphics.cpp:983-991``). +- ``screentexpos_flag`` per cell: ``GRAYSCALE``, ``ADDCOLOR``, + ``ANCHOR_SUBORDINATE``, ``TOP_OF_TEXT``, ``BOTTOM_OF_TEXT``, + ``ANCHOR_USE_SCREEN_COLOR`` (``graphics.h:2668-2677``). +- The same five arrays again for the **top layer** (``screen_top``, + ``screentexpos_top*``), composited over the base layer when + ``gps.top_in_use`` (e.g. announcements/tutorial overlays). +- Write order in ``update_all``: background → ``update_tile`` per cell → + ``update_anchor_tile`` → top-layer ``update_top_tile``/ + ``update_top_anchor_tile``. + +``texture_fullid`` (``enabler.h:646``/``671``) = ``{texpos, r,g,b, +br,bg,bb, flag}`` — the SDL renderer's cache key; ``flag`` is +``texture_fullid_flag`` (``TRANSPARENT_BACKGROUND``, ``DO_RECOLOR``, +``CONVERT``). + +Viewports and map ports +----------------------- + +- ``graphic_viewportst`` (``graphics.h:3319``) — a layered representation + of the map portion of the screen: ~23 parallel ``int32_t`` arrays + (``screentexpos``, ``_background``, ``_background_two``, + ``_floor_flag``, ``_liquid_flag``, ``_spatter_flag``, ``_spatter``, + ``_ramp_flag``, ``_shadow_flag``, ``_building_one``/``_two``, + ``_top_shadow``, ``_item``, ``_vehicle``, ``_projectile``, + ``_high_flow``, ``_vermin``, ``_signpost``, five ``*_creature`` + directions, ``_designation``, ``_interface``) plus a matching ``*_old`` + set for diffing. ``screen_x``/``screen_y`` are its private cursor; + ``clipx``/``clipy`` clip writes; ``flag`` has + ``GRAPHIC_VIEWPORT_FLAG_ACTIVE``. +- ``gps.main_viewport`` + ``gps.lower_viewport[LOWER_VIEWPORT_MAX]`` (=8) + — the multilevel z-view: lower viewports draw beneath the main one. + ``multilevel_depth_texpos[8]`` picks the depth shading tiles. +- ``graphic_map_portst`` (world-map viewports): base + 8 ``edge`` + 8 + ``edge2`` + detail/tunnel/river/road/site/army/interface + + ``detail_to_*`` (8 dirs) + ``site_to_s`` + ``cloud_bits`` (u64 bitmask + per cell) — plus ``_old`` twins, ``pixel_perc_x/y``, + ``top_left_corner_*``. +- Resizing flows through ``reshape_viewports(zoom_factor)`` → + ``shape_viewport_according_by_pixel_size`` (32px or + ``zoom_factor*32/128`` per tile; map ports use 16px). + +Per-frame lifecycle +------------------- + +1. Main thread ``do_frame``: ``render`` cmd → sim thread. +2. Sim thread: ``swap_arrays()`` — ``gps.*`` ↔ ``renderer.*`` pointer + exchange for every buffer, including each viewport's ``*_old`` arrays. +3. Sim thread: ``render_things()`` (``graphics.cpp:1199``): + + - walk ``gview.view`` ``child`` chain to topmost viewscreen; + - ``hooks_prerender()``; + - ``currentscreen->render(curtick)`` then + ``currentscreen->widgets.render(curtick)`` — **widgets render last, + deliberately "mostly DFHack"** (comment at line 1218). This is the + ordering DFHack's overlay relies on. + - REC/PLAY/prefix/FPS indicators. + +4. Sim acks ``complete``; main thread ``async_wait`` returns. +5. Main thread ``display()`` (``enabler.cpp:218``): if + ``force_full_display_count>0``, ``do_blank_screen_fill`` + full + viewport/map_port rebuilds; else per-cell diff between + ``screentexpos_*`` and ``*_old`` → ``update_viewport_tile``/ + ``update_map_port_tile`` only for changed cells. Then + ``update_all()`` → ``render()`` → ``SDL_RenderPresent``. + +The SDL renderer (``renderer_2d.hpp``) +-------------------------------------- + +``renderer_2d_base`` implements the ``renderer`` interface with an +``SDL_Renderer``: a ``texture_fullid → SDL_Texture*`` cache, +``update_tile`` / ``update_anchor_tile`` per-cell compositing (reading +``screen`` bytes + ``screentexpos`` + flags into a ``texture_fullid``, +looking it up in the cache, ``SDL_RenderCopy``), ``update_all`` (full +traversal), ``tidy_tile_cache`` (LRU-ish eviction), ``clean_tile_cache`` +(full flush — triggered via ``gps.do_clean_tile_cache``), ``zoom``/ +``resize``/``grid_resize``/``set_viewport_zoom_factor``/ +``set_fullscreen``. + +``screen_to_texid`` (``enabler.cpp:37``) is the canonical cell→texture +translation: char byte → font texpos (basic/large/small, or ``_top``/ +``_bot`` for half-height text rows), or ``screentexpos`` + +GRAYSCALE/ADDCOLOR color mixing. + +``init.font.*_font_texpos[256]`` (``init.h``, ``df.g_src.init.xml``) map +glyph bytes → font-page texposes, derived from ``curses_*.png`` font +sheets by ``prepare_graphics``. + +Input pipeline (``enabler_input.cpp``, ``keybindings.h``) +========================================================= + +SDL event → ``enabler.add_input`` (main thread, sim paused) → +``KeyEvent{release, EventMatch{type, mod, scancode, key|button|y}}`` → +``key_translation`` looks up ``EventMatch`` in the static ``keymap`` +(``multimap``) → ``Event``\ s go into the +``timeline`` (``set`` ordered by ``time`` then ``serial`` — one +physical press = one serial, possibly several ``InterfaceKey``\ s). + +Sim thread consumes via ``enabler_inputst::get_input(now)`` → +``std::set`` for one physical press; repeats are rescheduled +onto the timeline per ``repeatmap`` (``REPEAT_NOT``/``SLOW``/``FAST``, +tunable via ``init.input.hold_time``/``repeat_time``/``repeat_accel_*``). +``REPEAT_MWHEEL`` uses ``repeats`` as a countdown so one wheel notch +generates exactly N events. + +``InterfaceKey`` (``keybindings.h``, ~1800 constants, +``df.g_src.keybindings.xml``) is the binding vocabulary; +``bindingNames``/``displayNames``/``sdlNames``/``sdlUnicode`` bimaps +translate between keys and human strings. Macros record +``set`` lists (``prefs/macros/*.mak``), played back onto the +timeline. + +``interface.h`` adds ``standardscrolling``/``secondaryscrolling``/ +``standardstringentry`` helpers, ``INTERFACEFLAG_RETAIN_NONZERO_INPUT``, +and ``interfacest gview`` — the viewscreen stack root. + +Interface layer (``ViewBase.h``, ``interface.h``) +================================================= + +- ``viewscreenst`` (``ViewBase.h:1050``): ``child``/``parent`` links form + the screen stack (``gview.view`` is the root *member*, not a pointer). + ``breakdownlevel`` (``INTERFACE_BREAKDOWN_*``) requests teardown; + ``feed(set)``, ``logic()``, ``render(curtick)`` are the + three virtuals the loop calls; ``set_port_flags()`` lets a screen claim + viewports. +- ``interfacest::addscreen``/``removescreen`` splice the list + (``INTERFACE_PUSH_*``); ``interfacest::loop`` (``interface.cpp:870``) is + invoked from inside ``mainloop()`` and drives feed/logic/render per + screen (inferred — ``mainloop`` itself is closed source). +- ``viewscreenst::widgets`` (a ``widgets::container``) is the per-screen + widget tree — the DFHack overlay attaches widgets here. Container + types: ``container``, ``widget_stack``, ``rows_container``, + ``columns_container``, ``tabs``, ``scroll_rows``, ``radio_rows``, + ``table``, ``folder``, ``filter``, ``multifilter``, + ``params_container``; display widgets: ``text``, ``text_truncated``, + ``text_multiline``, ``nineslice``, ``anchored_tile``, + ``keybinding_display``, ``graphics_switcher``, ``character``; inputs: + ``interface_*_button``, ``better_button``, ``textbox``, ``dropdown``. + Anchoring is Godot-``Control``-style (``anchor_*`` fractions + + ``offset_*`` + ``LayoutPreset``\ s). + +Other modules DFHack touches +============================ + +- ``initst init`` — all init.txt state; ``init.display.grid_x``/ + ``grid_y`` are the *tile-grid* dimensions; ``init.display.flag`` has + ``USE_GRAPHICS``, ``TEXT``, ``NOT_RESIZABLE``, ``SOFTWARE``; + ``init.input.zoom_speed``, ``pause_zoom_no_interface_ms``. +- ``musicsoundst musicsound`` (``music_and_sound_g.h:422``) — song/ + ambience state machine (``Song``/``MusicEvent``/``MusicContext`` enums, + ``MUSICSOUND_FLAG_*``), up to ``SOUND_CHANNELNUM`` =32 channels; + ``update()`` is called from the main event loop; ``loading_files`` are + async futures. +- ``command_linest`` — ``game->command_line.original``, ``arg_vect``, + ``gen_id``/``world_seed``/``world_param``. +- ``random.h`` — ``trandom(limit)``, + ``push_trandom_uniform_seed``/``pop_trandom_uniform_seed`` (seed stack), + ``trandom_twist``. +- ``filest``/``stringvectst``/``textlinesst`` — path resolution across DF + dirs (``canon_location``/``both_locations``/ + ``with_flags(FILE_FLAG_*)``), manual ``char*`` string vector, raw-file + loader. +- ``LuaState`` (``lua.h``) — DF embeds Lua for the closed side (worldgen, + utterances); ``add_lua_df_globals`` registers ``trandom``, ``lua_log``, + ``capitalize_string_*``; ``data/init/globals.lua`` is run at state + init. Not mapped in df-structures. +- ``platform.h``/``win32_compat.cpp`` — Win32 names on Linux: + ``GetTickCount`` ≈ ``SDL_GetTicks``, ``MessageBox`` → + ``SDL_ShowMessageBox`` (with cross-thread marshaling via + ``show_message``), ``Sleep``, ``QueryPerformance*``, ``DWORD``, + ``MB_*``, ``VK_*``. Values are locally defined and may differ from real + Windows constants. + +Problematic / outdated / unsafe code +==================================== + +Confirmed defects and latent hazards. Severity tags: **[bug]** real +defect; **[ub]** undefined/unspecified behavior risk; **[fragile]** works +today but easily broken; **[stale]** dead/outdated code. + +Channels and threading +---------------------- + +- **[ub]** ``Chan::read`` waits ``cv.wait(lk)`` with no predicate; a + spurious wakeup (or a wakeup for a message another consumer took) makes + ``vals.front()``/``pop()`` run on an empty queue — UB + (``mail.hpp:34-40``). +- **[ub]** ``Chan::try_read_for_millis`` treats *any* ``wait_for`` return + of ``no_timeout`` as "queue non-empty" and pops unconditionally + (``mail.hpp:23-33``). Also only one consumer can safely read a channel; + ``try_read`` races multiple consumers by design of ``try_to_lock``. +- **[fragile]** ``Chan::write(const T &w)`` calls + ``vals.push(std::move(w))`` — moving from a const ref copies; the API + looks like it moves but doesn't (``mail.hpp:41-45``). +- **[fragile]** ``async_cmd``/``async_msg`` have a union + tag and their + ctors don't initialize the union (``enabler.h:875-899``); a reader using + the wrong member reads garbage. ``async_wait`` dispatches on ``msg`` so + this is currently consistent — but ``async_msg(show_message)`` leaves + ``fps``/``x``/``y`` uninitialized and vice versa. +- **[bug/ub]** ``enabler.flag``, ``enabler.async_paused``, ``simticks``, + ``gputicks``, ``async_frames``, ``gps.mouse_*`` are non-atomic and + cross-thread (e.g. ``flag`` write ``enabler.cpp:1005`` vs read ``784``; + ``async_paused`` write ``enabler.cpp:1219`` vs read ``757``/``800``; + ``simticks`` read on main in ``enabler_input.cpp:703``/``741``). Mostly + benign on x86, formally racy. +- **[fragile]** ``async_msg::show_message`` ships raw ``const wchar_t*`` + cross-thread; lifetime is guarded only by the sender blocking on + ``async_fromcomplete``. If anything drains ``async_frombox`` without + servicing ``show_message``, or releases the semaphore early, UAF. +- **[fragile]** ``release_grid_size`` drains *all* + ``overridden_grid_sizes`` then sends one ``zoom_resetgrid``; the + in-source FIXME notes a known push/pop mismatch ("overriding once per + movie but apparently only releasing for the last one", + ``enabler.cpp:1157``). +- **[stale]** ``enabler_input.cpp:677`` calls ``enabler.simticks.load()`` + — but ``simticks`` is a plain ``int``. This is inside ``#ifdef CURSES``; + the curses input path cannot compile as written. Indicates the curses + port bit-rotted after ``simticks`` stopped being atomic. + +Screen-buffer writes +-------------------- + +- **[bug]** ``addchar_flag(x, y, ...)`` and ``top_addchar_flag(x, y, + ...)`` write ``screentexpos_flag[screenx*dimy + screeny]`` — the + *cursor* cell, not the ``(x,y)`` argument cell (``graphics.h:11183``, + ``11309``). +- **[bug]** ``addchar(x, y, c, f, b, bright)`` (and ``top_addchar``, + ``addchar_flag``, ``top_addchar_flag`` ``(x,y)`` overloads) compute + ``fg = f + (bright ? 8 : 0)`` and then ignore it, indexing + ``uccolor[f]`` — the bright flag is dropped on these overloads + (``graphics.h:11112-11118``). +- **[ub]** the ``(x,y)`` ``addchar``/``top_addchar``/``addchar_flag``/ + ``top_addchar_flag`` overloads index ``uccolor[f]``/``uccolor[b]`` with + raw u8 parameters — any color index >15 is an out-of-bounds read + (``graphics.h:11113-11118`` etc.). +- **[fragile]** ``locate``+``addchar`` bounds-safety relies on the clip + rect: the pointer check ``s < screen_limit`` has no lower bound in the + cursor-based overloads; a negative ``screenx`` passes the check and is + only saved because ``clipx[0] >= 0``. Calling ``setclipping`` with + negative values breaks that invariant. +- **[bug]** ``gps_locator::is_free`` reads ``gps.screen[x*dimy*8 + + y*8]`` with no bounds check at all (``graphics.h:11433+``). + Over/underflow reads if called with out-of-range coordinates. +- **[fragile]** ``graphicst::dim_colors``, ``rain/snow/color_square`` + operate on the 8-byte cell layout with magic offsets; several are only + correct for the 16-color palette path. +- **[stale]** ``graphicst::gray_out_rect`` is fully commented out with a + note that ``*4`` should be ``*8`` — the 4-byte cell era ended long ago + (``graphics.h:11351-11367``). The TTF path in ``addst``/``top_addst`` + (``GRAPHICSTYPE_TTF``, ``ttf_manager``, ``ttfstr``) is similarly dead — + commented out wholesale (``graphics.cpp:482-530``, ``562-610``). + +Renderer +-------- + +- **[fragile]** ``renderer_2d_base::update_all`` uses hard-coded display + numbers (1920/1080/907) and ``SDL_GetDisplayBounds(0, ...)`` on display + 0 for title-screen/background sizing (``renderer_2d.hpp`` + ``update_all``). +- **[fragile]** ``renderer_2d_base::zoom``: ``case zoom_reset:`` sets + ``zoom_steps = 0`` then **falls through** into ``zoom_resetgrid`` → + ``compute_forced_zoom()`` (``renderer_2d.hpp:4478-4481``). Probably + intended — reset implies recompute — but there is no + ``[[fallthrough]]`` or comment. +- **[bug]** ``renderer_2d_base`` zoom/reshape math has in-source FIXMEs: + "perc introduces some error so at certain zooms it is way off (the -16 + is a cludge)" (``graphics.cpp:306-311``) — map-port origin math is + approximate and known-wrong at some zoom levels. +- **[fragile]** ``clean_cached_tile``/``clean_tile_cache``/ + ``tidy_tile_cache`` call ``SDL_DestroyTexture`` on cache entries; + entries can be null — tolerated by SDL2 but unchecked. +- **[stale]** ``renderer`` declares ``directtexcopy``/ + ``directtexcopy_old`` buffers that ``gps_allocate`` never allocates — + dead fields still mapped in ``df.g_src.enabler.xml`` + (``enabler.h:750``/``772``). +- **[stale]** ``renderer_opengl.hpp`` is ``#include``\ d out + (``enabler.cpp:194``) and ``loop()`` has a block of commented + ``renderer_opengl``/``partial``/… selection logic + (``enabler.cpp:1101-1116``); ``renderer_curses`` is ``#ifdef CURSES`` + -only. ``ttf_manager`` exists but its users are commented out. +- **[fragile]** ``renderer_2d`` ctor exits the process on init failure + (window/renderer creation) rather than reporting. + +Input +----- + +- **[fragile]** ``add_input`` synthesizes release/re-press ``KeyEvent``\ s + for all pressed keys on every modifier change + (``enabler_input.cpp:537-555``) — quirky by design; DFH code that + injects ``KeyEvent``\ s should use ``add_input_refined`` ("Use this one. + It's much nicer.", ``enabler_input.h:96``). +- **[fragile]** ``load_macro_from_file`` uses fixed 100-char ``getline`` + buffers and hand-rolled parsing; malformed files print to cout and can + partially populate ``macros`` (``enabler_input.cpp:968-1043``). +- **[bug-ish]** ``get_input``'s "3 simticks before first repeat" guard + can drop a repeat event silently (``enabler_input.cpp:782-785``). +- **[stale]** ``GetKeyDisplay`` prints to ``cout`` on missing binding and + returns "?" — fine for dev, useless in production. + +Platform layer and misc +----------------------- + +- **[bug]** ``win32_compat.cpp::MessageBox``: ``new char[64]``/ + ``new char[128]`` for title/text are **leaked on every call** — the + heap allocation is unnecessary (``SDL_ShowMessageBox`` is synchronous + and copies the strings; the comment claiming stack pointers are unsafe + is wrong) (``win32_compat.cpp:125-129``). ``wcstombs`` also truncates + at fixed sizes and loses non-ASCII. +- **[fragile]** ``platform.h`` redefines Win32 constants (``MB_*``, + ``VK_*``, ``DWORD``, etc.) by hand; several values are explicitly noted + not to match Windows. Code compiled against real Win32 values may + behave differently. +- **[stale]** ``endian.h`` byteswaps only under ``__ppc__``/ + ``__ppc64__`` — dead on all modern targets. +- **[bug-ish]** ``stringvectst`` (``files.h``/``basics.h``) manually owns + ``char*`` strings, erases from the front, and leaks if methods are used + in unexpected order; several methods ignore allocation failure. +- **[fragile]** file/decompression helpers (``files.cpp``, + ``basics.cpp``) read sizes and resize buffers from on-disk values + without validation — malformed data can trigger huge allocations or + overruns. +- **[fragile]** ``texture_handlerst``/``palettest``/``textures`` do + manual ``new[]``/``delete[]`` and pointer bookkeeping; error paths leak + or leave dangling pointers (e.g. ``clone_texture`` pushes a NULL + surface into ``raws``, ``textures.cpp:59-71``). +- **[fragile]** ``lua.cpp``: ``lua_push_flags_member`` calls + ``std::tolower`` on possibly-signed ``char`` (UB for negative chars); + several raw ``lua_*`` stack/index manipulations assume fixed stack + positions. +- **[stale]** ``enabler.h`` still carries OpenGL-era members/comments + (``reset_textures`` semantics predate SDL_Renderer; "OpenGL state + (wrappers)" comment on ``textures``), and ``enabler.cpp``/ + ``graphics.cpp`` have extensive commented-out TTF/OpenGL/partial-print + code. +- **[fragile]** ``musicsoundst`` mixes raw ints/``DWORD``\ s, + ``svector``\ s, and ``std::future``\ s; ``SOUND_CHANNELNUM``-indexed + arrays in ``interfacest`` movie code are fixed at 200 entries with no + bounds checks (``interface.h:115``). +- **[fragile]** ``emit_logs``/``errorlog_string``/``gamelog_string`` + open/close file streams per call plus keep static ``ofstream``\ s — the + comment admits "opening/closing these too often seems to cause crashes + on linux" (``basics.cpp:54-56``). + +Quick map: g_src symbol → DFHack +================================= + +.. list-table:: + :header-rows: 1 + + * - g_src + - df-structures / DFHack + * - ``enabler`` (global ``enablerst``) + - ``df.global.enabler`` (``df/enabler.h``) + * - ``gps`` (``graphicst``) + - ``df.global.gps`` (``df/graphic.h``) + * - ``gview`` (``interfacest``) + - ``df.global.gview`` (``df/interface.h``); + ``Gui::getCurViewscreen`` + * - ``init`` (``initst``) + - ``df.global.init`` + * - ``musicsound`` + - ``df.global.musicsound`` + * - ``mainloop()`` + - not mapped — closed side; reached via ``Core::Update`` inside + ``dfhooks_update`` + * - ``hooks_*`` trampolines + - ``dfhooks_*`` exports in ``library/Hooks.cpp`` + * - ``renderer`` vtable + - ``df/renderer.h`` vmethods — interposable via ``VTableInterpose``; + the old ``Renderer::renderer_wrap`` chaining code in + ``library/modules/Renderer.cpp`` is entirely commented out + * - ``InterfaceKey`` + - ``df.interface_key`` enum; ``Screen``/``Gui`` modules + * - ``enabler.async_tobox``/``async_frombox``/``async_zoom`` + - modeled inline in ``df.g_src.enabler.xml`` (mutex+condvar+deque + members) + * - ``screentexpos*`` arrays + - ``df.gps``/``df.renderer`` pointer fields; + ``Screen::paintScreen``/``df.screen`` diff --git a/docs/dev/index.rst b/docs/dev/index.rst index 8a402ee03a..e2d1905dfd 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -17,6 +17,7 @@ These are pages relevant to people developing for DFHack. /docs/dev/Lua API /docs/dev/overlay-dev-guide /docs/dev/Structures-intro + /docs/dev/g_src-notes /docs/dev/CoreSuspender /docs/dev/data-identity /docs/dev/github-workflows From a69120f3042aebb21ae6a7caa9ad60b5c9c9b477 Mon Sep 17 00:00:00 2001 From: Dan Muller Date: Tue, 20 Oct 2026 10:38:23 -0400 Subject: [PATCH 04/42] RemoteFortressReader: convert creature/caste names and descriptions to UTF-8 in GetPartialCreatureRaws Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/changelog.txt | 1 + .../remotefortressreader.cpp | 30 +++++++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/changelog.txt b/docs/changelog.txt index c2470dbe87..972d1c6527 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -60,6 +60,7 @@ Template for new versions: ## New Features ## Fixes +- ``RemoteFortressReader``: ensured creature and caste names and caste descriptions in ``GetCreatureRaws``/``GetPartialCreatureRaws`` are transmitted in UTF-8 instead of CP437 ## Misc Improvements diff --git a/plugins/remotefortressreader/remotefortressreader.cpp b/plugins/remotefortressreader/remotefortressreader.cpp index 9a8f034c5b..3145ca1add 100644 --- a/plugins/remotefortressreader/remotefortressreader.cpp +++ b/plugins/remotefortressreader/remotefortressreader.cpp @@ -2626,15 +2626,15 @@ static command_result GetPartialCreatureRaws(color_ostream &stream, const ListRe send_creature->set_index(i); send_creature->set_creature_id(orig_creature->creature_id); - send_creature->add_name(orig_creature->name[0]); - send_creature->add_name(orig_creature->name[1]); - send_creature->add_name(orig_creature->name[2]); + send_creature->add_name(DF2UTF(orig_creature->name[0])); + send_creature->add_name(DF2UTF(orig_creature->name[1])); + send_creature->add_name(DF2UTF(orig_creature->name[2])); - send_creature->add_general_baby_name(orig_creature->general_baby_name[0]); - send_creature->add_general_baby_name(orig_creature->general_baby_name[1]); + send_creature->add_general_baby_name(DF2UTF(orig_creature->general_baby_name[0])); + send_creature->add_general_baby_name(DF2UTF(orig_creature->general_baby_name[1])); - send_creature->add_general_child_name(orig_creature->general_child_name[0]); - send_creature->add_general_child_name(orig_creature->general_child_name[1]); + send_creature->add_general_child_name(DF2UTF(orig_creature->general_child_name[0])); + send_creature->add_general_child_name(DF2UTF(orig_creature->general_child_name[1])); send_creature->set_creature_tile(orig_creature->creature_tile); send_creature->set_creature_soldier_tile(orig_creature->creature_soldier_tile); @@ -2654,15 +2654,15 @@ static command_result GetPartialCreatureRaws(color_ostream &stream, const ListRe send_caste->set_caste_id(orig_caste->caste_id); - send_caste->add_caste_name(orig_caste->caste_name[0]); - send_caste->add_caste_name(orig_caste->caste_name[1]); - send_caste->add_caste_name(orig_caste->caste_name[2]); + send_caste->add_caste_name(DF2UTF(orig_caste->caste_name[0])); + send_caste->add_caste_name(DF2UTF(orig_caste->caste_name[1])); + send_caste->add_caste_name(DF2UTF(orig_caste->caste_name[2])); - send_caste->add_baby_name(orig_caste->baby_name[0]); - send_caste->add_baby_name(orig_caste->baby_name[1]); + send_caste->add_baby_name(DF2UTF(orig_caste->baby_name[0])); + send_caste->add_baby_name(DF2UTF(orig_caste->baby_name[1])); - send_caste->add_child_name(orig_caste->child_name[0]); - send_caste->add_child_name(orig_caste->child_name[1]); + send_caste->add_child_name(DF2UTF(orig_caste->child_name[0])); + send_caste->add_child_name(DF2UTF(orig_caste->child_name[1])); send_caste->set_gender(orig_caste->sex); for (size_t partIndex = 0; partIndex < orig_caste->body_info.body_parts.size(); partIndex++) @@ -2781,7 +2781,7 @@ static command_result GetPartialCreatureRaws(color_ostream &stream, const ListRe } } - send_caste->set_description(orig_caste->description); + send_caste->set_description(DF2UTF(orig_caste->description)); send_caste->set_adult_size(orig_caste->misc.adult_size); } From e4e54591ef4a020c9321f1e0a812f2cb7f199db1 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Tue, 20 Oct 2026 22:19:18 -0500 Subject: [PATCH 05/42] corrections --- docs/dev/g_src-notes.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/dev/g_src-notes.rst b/docs/dev/g_src-notes.rst index e7c1a5e221..05de2a62e2 100644 --- a/docs/dev/g_src-notes.rst +++ b/docs/dev/g_src-notes.rst @@ -52,7 +52,7 @@ Source inventory and XML correspondence - ``renderer_curses`` (PRINT_MODE:TEXT) - ``df.g_src.renderer_2d.xml`` * - ``renderer_offscreen.cpp`` - - offscreen renderer for benchmarking + - offscreen renderer used for map renders (in olders releases, functionality currently not available) - ``df.g_src.renderer_2d.xml`` * - ``renderer_opengl.hpp`` - legacy OpenGL renderer (not compiled — ``#include`` commented out in @@ -583,7 +583,7 @@ Other modules DFHack touches ``trandom_twist``. - ``filest``/``stringvectst``/``textlinesst`` — path resolution across DF dirs (``canon_location``/``both_locations``/ - ``with_flags(FILE_FLAG_*)``), manual ``char*`` string vector, raw-file + ``with_flags(FILE_FLAG_*)``), manual ``std::string *`` vector (implemented via the ``pstringst`` class), raw-file loader. - ``LuaState`` (``lua.h``) — DF embeds Lua for the closed side (worldgen, utterances); ``add_lua_df_globals`` registers ``trandom``, ``lua_log``, @@ -738,8 +738,8 @@ Platform layer and misc - **[stale]** ``endian.h`` byteswaps only under ``__ppc__``/ ``__ppc64__`` — dead on all modern targets. - **[bug-ish]** ``stringvectst`` (``files.h``/``basics.h``) manually owns - ``char*`` strings, erases from the front, and leaks if methods are used - in unexpected order; several methods ignore allocation failure. + C++ STL strings, erases from the front, and can leak if a consumer directly accesses its public + data element or if ``read_file`` is used on a non-empty list. Several methods ignore allocation failure. - **[fragile]** file/decompression helpers (``files.cpp``, ``basics.cpp``) read sizes and resize buffers from on-disk values without validation — malformed data can trigger huge allocations or From 8e6026766f9975d9fdfc001dcf63e443dd770864 Mon Sep 17 00:00:00 2001 From: SilasD Date: Wed, 21 Oct 2026 15:43:08 -0700 Subject: [PATCH 06/42] Units.cpp: mothers now have flags4.bits.carrying_somebody flags --- library/modules/Units.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/modules/Units.cpp b/library/modules/Units.cpp index a5472b4e95..d661f8b3ab 100644 --- a/library/modules/Units.cpp +++ b/library/modules/Units.cpp @@ -860,7 +860,7 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) unit->idle_area = target_pos; // Move unit's riders (including babies) to destination - if (unit->flags1.bits.ridden) + if (unit->flags1.bits.ridden || unit->flags4.bits.carrying_somebody) for (auto rider : world->units.other[units_other_id::ANY_RIDER]) if (rider->relationship_ids[unit_relationship_type::RiderMount] == unit->id) rider->pos = unit->pos; From 90845252a3c46860971580737348a0377bcac61b Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Thu, 22 Oct 2026 16:35:44 -0500 Subject: [PATCH 07/42] documentation on DF materials --- docs/dev/Material-encoding.rst | 194 +++++++++++++++++++++++++++++++++ docs/dev/index.rst | 1 + 2 files changed, 195 insertions(+) create mode 100644 docs/dev/Material-encoding.rst diff --git a/docs/dev/Material-encoding.rst b/docs/dev/Material-encoding.rst new file mode 100644 index 0000000000..d9129e5809 --- /dev/null +++ b/docs/dev/Material-encoding.rst @@ -0,0 +1,194 @@ +.. _material-encoding: + +##################################################### +How DF encodes materials: ``mat_type``/``mat_index`` +##################################################### + +Almost everywhere that Dwarf Fortress needs to refer to a material -- items, +item improvements, job requirements, plants, reactions, syndromes, map +features, and so on -- it does so with a pair of integers. In the names +provided by Bay 12, the pair is usually called ``material`` and +``matgloss``, although other naming is sometimes seen, such as ``subcat1``/ +``subcat2``. Before we learned otherwise, the DFHack team originally called +what we now know to be a matgloss a ``mat_index`` ("material index"), and so +this term tends to be used in DFHack code instead of "matgloss". Thus, you may +see these two terms used interchangeably in DFHack code, in documentation, and +in community discussions. + +.. contents:: Contents + :local: + +The ``mat_type`` half +===================== + +``mat_type`` is an index into ``world.raws.mat_table.builtin`` +(a ``special_mat_table``, DF name ``material_infost.misc_def``), which is a +static array of pointers to ``df::material`` objects indexed by the +``builtin_mats`` enum (DF name ``MaterialType``). The enum is laid out as +follows: + +================= ===================== ============================================= +Numeric range Enum values Meaning +================= ===================== ============================================= +-1 ``NONE`` No material / invalid +0 ``INORGANIC`` ``mat_index`` selects a ``df::inorganic_raw`` +1..18 ``AMBER``..``GRIME`` True built-in materials +19..218 ``CREATURE_1..200`` Creature raw materials +219..418 ``HIST_FIG_1..200`` Historical figure creature materials +419..618 ``PLANT_1..200`` Plant raw materials +619..658 ``UNUSED01..40`` Unused; treated as built-ins, normally null +================= ===================== ============================================= + +The named built-ins are ``AMBER``, ``CORAL``, ``GLASS_GREEN``, +``GLASS_CLEAR``, ``GLASS_CRYSTAL``, ``WATER``, ``COAL``, ``POTASH``, +``ASH``, ``PEARLASH``, ``LYE``, ``MUD``, ``VOMIT``, ``SALT``, ``FILTH_B`` +(feces), ``FILTH_Y`` (urine), ``UNKNOWN_SUBSTANCE`` (semen), and ``GRIME``. +Several of these built-ins exist solely for legacy purposes and do not normally +arise in unmodded games; for example, ``FILTH_B`` and ``FILTH_Y`` only appear +in very specific situations, and ``UNKNOWN_SUBSTANCE`` was only ever seen in +connection with tentacle demons, which haven't been in the game since version +0.28.181.40d. These are materials constructed by the game itself (not loaded +from raws). The array still has a slot for every enum value, but entries outside +the named built-ins are not real materials -- for the ``CREATURE_*``, +``HIST_FIG_*``, and ``PLANT_*`` ranges the actual ``df::material`` lives +inside a raw object and ``mat_index`` is needed to find it. Entries in +the array may be null, so consumers should always check the resolved +pointer. + +The ``mat_index`` half +====================== + +The meaning of ``mat_index`` depends entirely on the ``mat_type`` range: + +* ``mat_type`` < 0 (``NONE``): no material. ``mat_index`` should also be -1. + +* ``mat_index`` == -1, any non-negative ``mat_type``: a "bare" reference to + the built-in material object ``mat_table.builtin[mat_type]``. For most + built-ins this is just the material itself, but ``(0, -1)`` is special: + it is the *inorganic wildcard*, meaning "any inorganic material", used in + contexts such as job item constraints (e.g. "any stone"). + +* ``mat_type`` == 0 (``INORGANIC``), ``mat_index`` >= 0: ``mat_index`` is + an index into ``world.raws.inorganics.all`` (a ``df::inorganic_raw``), + and the material is ``inorganic->material``. For example, + ``(0, )`` is iron. + +* 1 <= ``mat_type`` <= 18 (built-ins), ``mat_index`` >= 0: still a + built-in material; ``mat_index`` is normally ignored. The one exception + is ``COAL`` (7), where ``mat_index`` distinguishes the product: + 0 means coke and 1 means charcoal. + +* 19 <= ``mat_type`` <= 218 (``CREATURE_1``..``CREATURE_200``): + ``mat_index`` is an index into ``world.raws.creatures.all`` + (a ``df::creature_raw``), and ``mat_type - CREATURE_1`` is an index into + that creature's ``material`` vector. In other words, ``mat_type`` picks + both the table *and* which of the creature's (up to 200) defined + materials is meant, while ``mat_index`` picks the creature. + For example, dwarf bone is + ``(CREATURE_1 + , )``. + +* 219 <= ``mat_type`` <= 418 (``HIST_FIG_1``..``HIST_FIG_200``): like the + creature range, except that ``mat_index`` is a **historical figure id**, + not a vector index -- the figure is looked up by its ``id`` field in + ``world.history.figures``. The material itself is + ``creature->material[mat_type - HIST_FIG_1]`` of the figure's race + (``creature_raw::find(figure->race)``). These pairs identify materials + belonging to a *specific* historical figure, e.g. products of named + creatures such as forgotten beasts, titans, and demons. + +* 419 <= ``mat_type`` <= 618 (``PLANT_1``..``PLANT_200``): + ``mat_index`` is an index into ``world.raws.plants.all`` + (a ``df::plant_raw``), and ``mat_type - PLANT_1`` is an index into that + plant's ``material`` vector. + +* ``mat_type`` > 618 (``UNUSED*``): treated as a built-in; + ``mat_table.builtin[mat_type]`` is used directly and is normally null. + +Where the materials themselves live +=================================== + +All of the above resolves to a ``df::material`` object (DF name +``material_definitionst``), which holds the material id, per-state names +and colors, flags, heat properties, and so on: + +* Built-ins: ``world.raws.mat_table.builtin[mat_type]``. +* Inorganics: embedded in ``world.raws.inorganics.all[mat_index]->material``. +* Creatures: ``world.raws.creatures.all[mat_index]->material[subtype]``. +* Plants: ``world.raws.plants.all[mat_index]->material[subtype]``. + Additionally, each ``df::plant_raw`` has a ``material_defs`` block -- + parallel ``type``/``idx`` arrays (DF names ``use_mat``/``use_matg``) + indexed by ``plant_material_def`` -- that records the canonical + ``(mat_type, mat_index)`` pair for each defined use of the plant + (``basic_mat``, ``tree``, ``drink``, ``seed``, ``thread``, ``mill``, + ``extract_vial``, ``extract_barrel``, ``extract_still_vial``). + +Where the pairs appear +====================== + +The ``mat_type``/``mat_index`` naming convention is used fairly consistently +in DF structures for individual fields, so ``MaterialInfo::decode`` can +consume any structure that exposes those members. Some representative +examples: + +* Most item types (``item_barst``, ``item_woodst``, ``item_boulderst``, + ``item_threadst``, etc.) store the pair directly in ``mat_type`` and + ``mat_index`` fields. +* ``item_body_component`` (corpses and body parts) instead stores + ``race``, ``caste``, ``hist_figure_id``, and ``unit_id``. The item + virtual methods ``getActualMaterial()`` and ``getActualMaterialIndex()`` + (DF names ``get_forced_material``/``get_forced_matgloss``) compute the + proper pair -- "an actual material, never a race/caste" -- so + ``MaterialInfo::decode(item)`` always uses those methods rather than the + raw fields. +* ``material_vec_ref`` is a DFHack convenience compound (not a real DF + structure) that bundles the parallel ``mat_type``/``mat_index`` vectors + DF uses for multi-material lists, e.g. in reaction products and + ``world.raws.mat_table.syndromes``/``effects``. In the actual Bay 12 + headers, these structures are parallel arrays appearing as sequential + members within a larger compound. +* ``job_item`` (work order and job requirements) uses ``mat_type``/ + ``mat_index`` for the required material, where the ``(0, -1)`` inorganic + wildcard is common. +* Historical-figure materials appear in things like body part items of + named creatures. + +Other names for the pair in DF code include ``material``/``matgloss`` +(items, improvements), ``use_mat``/``use_matg`` (plant material +definitions), ``itemneeded_material``/``itemneeded_matgloss`` (job items), +``dye_material``/``dye_matgloss`` (dyed improvements), +``hardens_with_water_mat``/``hardens_with_water_matg``, and +``syndrome_mat``/``syndrome_matg`` (syndrome tables). + +Working with the encoding in DFHack +=================================== + +``DFHack::MaterialInfo`` (exposed to Lua as ``dfhack.matinfo``) is the +canonical decoder for a pair. Constructing one via ``decode(type, index)``, +``decode(item)``, ``decode(material_vec_ref, idx)``, or +``decode(some_struct_pointer)`` fills in: + +* ``type``/``index``: the raw pair. +* ``material``: the resolved ``df::material`` pointer (``isValid()`` is + false when it is null). +* ``mode``: one of ``None``, ``Builtin``, ``Inorganic``, ``Creature``, or + ``Plant`` -- note that historical figure materials report ``Creature``, + distinguished by a non-null ``figure``. +* ``subtype``: the position within the creature/plant material vector. +* ``inorganic``, ``creature``, ``plant``: the resolved raw object for the + matching mode. +* ``figure``: the ``df::historical_figure`` for ``HIST_FIG_*`` pairs. + +``MaterialInfo::find`` and ``dfhack.matinfo.find`` also parse the +string-token forms produced by ``getToken()``: +``INORGANIC:`` (e.g. ``INORGANIC:IRON``), +``CREATURE::`` (e.g. ``CREATURE:DWARF:BONE``), +``PLANT::``, plain ```` for built-ins, and the +special cases ``COAL:COKE``/``COAL:CHARCOAL`` and ``NONE``. + +For plain integer pairs, ``DFHack::t_matpair`` and +``DFHack::t_matglossPair`` are simple ``{mat_type, mat_index}`` / +``{type, index}`` bundles, and ``isAnyInorganic()``/``isInorganicWildcard()`` +test for ``mat_type == 0`` and the ``(0, -1)`` wildcard respectively. + +See the `lua-api` documentation for the Lua-side ``dfhack.matinfo`` +interface. diff --git a/docs/dev/index.rst b/docs/dev/index.rst index 8a402ee03a..bac24b1a46 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -17,6 +17,7 @@ These are pages relevant to people developing for DFHack. /docs/dev/Lua API /docs/dev/overlay-dev-guide /docs/dev/Structures-intro + /docs/dev/Material-encoding /docs/dev/CoreSuspender /docs/dev/data-identity /docs/dev/github-workflows From 3940da2acdb8b47d249b99cb4d367d4ba0210d37 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Thu, 22 Oct 2026 16:46:18 -0500 Subject: [PATCH 08/42] fix whitespace --- docs/dev/Material-encoding.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/Material-encoding.rst b/docs/dev/Material-encoding.rst index d9129e5809..66f591583d 100644 --- a/docs/dev/Material-encoding.rst +++ b/docs/dev/Material-encoding.rst @@ -53,7 +53,7 @@ the named built-ins are not real materials -- for the ``CREATURE_*``, ``HIST_FIG_*``, and ``PLANT_*`` ranges the actual ``df::material`` lives inside a raw object and ``mat_index`` is needed to find it. Entries in the array may be null, so consumers should always check the resolved -pointer. +pointer. The ``mat_index`` half ====================== From 012ff52fcdef73243a87233fe68c70dec0fc9b89 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Fri, 23 Oct 2026 15:46:00 -0500 Subject: [PATCH 09/42] revision based on comments received --- docs/dev/Material-encoding.rst | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/docs/dev/Material-encoding.rst b/docs/dev/Material-encoding.rst index 66f591583d..e2506293d5 100644 --- a/docs/dev/Material-encoding.rst +++ b/docs/dev/Material-encoding.rst @@ -9,7 +9,10 @@ item improvements, job requirements, plants, reactions, syndromes, map features, and so on -- it does so with a pair of integers. In the names provided by Bay 12, the pair is usually called ``material`` and ``matgloss``, although other naming is sometimes seen, such as ``subcat1``/ -``subcat2``. Before we learned otherwise, the DFHack team originally called +``subcat2``. The ``subcat1``/``subcat2`` names usually appear on fields +that can hold *either* a material+matgloss pair *or* a race+caste pair, +with a companion ``item_type`` field determining how to interpret them. +Before we learned otherwise, the DFHack team originally called what we now know to be a matgloss a ``mat_index`` ("material index"), and so this term tends to be used in DFHack code instead of "matgloss". Thus, you may see these two terms used interchangeably in DFHack code, in documentation, and @@ -66,7 +69,10 @@ The meaning of ``mat_index`` depends entirely on the ``mat_type`` range: the built-in material object ``mat_table.builtin[mat_type]``. For most built-ins this is just the material itself, but ``(0, -1)`` is special: it is the *inorganic wildcard*, meaning "any inorganic material", used in - contexts such as job item constraints (e.g. "any stone"). + contexts such as job item constraints (e.g. "any stone"). The same + ``(INORGANIC, -1)`` pair -- written ``INORGANIC:NONE`` in token form -- + is also used for the "magma" material of liquid items picked up in + minecarts. * ``mat_type`` == 0 (``INORGANIC``), ``mat_index`` >= 0: ``mat_index`` is an index into ``world.raws.inorganics.all`` (a ``df::inorganic_raw``), @@ -76,7 +82,10 @@ The meaning of ``mat_index`` depends entirely on the ``mat_type`` range: * 1 <= ``mat_type`` <= 18 (built-ins), ``mat_index`` >= 0: still a built-in material; ``mat_index`` is normally ignored. The one exception is ``COAL`` (7), where ``mat_index`` distinguishes the product: - 0 means coke and 1 means charcoal. + 0 means coke and 1 means charcoal. DF has a dedicated enum for these + values, ``MatGlossCoal`` (``MATGLOSS_COAL_COKE`` = 0, + ``MATGLOSS_COAL_CHARCOAL`` = 1), though df-structures does not currently + expose it. * 19 <= ``mat_type`` <= 218 (``CREATURE_1``..``CREATURE_200``): ``mat_index`` is an index into ``world.raws.creatures.all`` @@ -93,8 +102,10 @@ The meaning of ``mat_index`` depends entirely on the ``mat_type`` range: ``world.history.figures``. The material itself is ``creature->material[mat_type - HIST_FIG_1]`` of the figure's race (``creature_raw::find(figure->race)``). These pairs identify materials - belonging to a *specific* historical figure, e.g. products of named - creatures such as forgotten beasts, titans, and demons. + belonging to a *specific* historical figure. In practice they are most + often seen in body-material spatters (``spatter_common``, e.g. blood) + left by named creatures, and in products of named creatures such as + forgotten beasts, titans, and demons. * 419 <= ``mat_type`` <= 618 (``PLANT_1``..``PLANT_200``): ``mat_index`` is an index into ``world.raws.plants.all`` @@ -139,7 +150,9 @@ examples: (DF names ``get_forced_material``/``get_forced_matgloss``) compute the proper pair -- "an actual material, never a race/caste" -- so ``MaterialInfo::decode(item)`` always uses those methods rather than the - raw fields. + raw fields. This is the dual-purpose encoding mentioned above: the raw + ``getMaterial()``/``getMaterialIndex()`` methods (DF ``get_subcat1``/ + ``get_subcat2``) return the race and caste for items like these. * ``material_vec_ref`` is a DFHack convenience compound (not a real DF structure) that bundles the parallel ``mat_type``/``mat_index`` vectors DF uses for multi-material lists, e.g. in reaction products and @@ -153,9 +166,9 @@ examples: named creatures. Other names for the pair in DF code include ``material``/``matgloss`` -(items, improvements), ``use_mat``/``use_matg`` (plant material -definitions), ``itemneeded_material``/``itemneeded_matgloss`` (job items), -``dye_material``/``dye_matgloss`` (dyed improvements), +(items, improvements, contaminants), ``use_mat``/``use_matg`` (plant +material definitions), ``itemneeded_material``/``itemneeded_matgloss`` +(job items), ``dye_material``/``dye_matgloss`` (dyed improvements), ``hardens_with_water_mat``/``hardens_with_water_matg``, and ``syndrome_mat``/``syndrome_matg`` (syndrome tables). From 285c385efd810c297971c5b59b7843fd04bf7fdd Mon Sep 17 00:00:00 2001 From: DFHack-Urist via GitHub Actions <63161697+DFHack-Urist@users.noreply.github.com> Date: Tue, 27 Oct 2026 13:01:23 +0000 Subject: [PATCH 10/42] Auto-update submodules plugins/stonesense: master --- plugins/stonesense | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/stonesense b/plugins/stonesense index 42e04ca70e..b71216ddfc 160000 --- a/plugins/stonesense +++ b/plugins/stonesense @@ -1 +1 @@ -Subproject commit 42e04ca70e2e08c1fc1cef88837a7b7a38935490 +Subproject commit b71216ddfc54e20ad48fb26f9ba1d82c7656b6ec From 9c779a04656b59b16ee667499f39bf9243d62c38 Mon Sep 17 00:00:00 2001 From: SilasD Date: Wed, 28 Oct 2026 07:48:57 -0700 Subject: [PATCH 11/42] Units.cpp add special handling for wagon riders. wagon riders do not ride the center tile of the wagon, and the draft animals are not even in the wagon's extent. I like being able to do math on coords. --- library/modules/Units.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/library/modules/Units.cpp b/library/modules/Units.cpp index d661f8b3ab..3d1a903797 100644 --- a/library/modules/Units.cpp +++ b/library/modules/Units.cpp @@ -31,6 +31,7 @@ distribution. #include "VersionInfo.h" #include "modules/Items.h" +#include "modules/Job.h" #include "modules/Maps.h" #include "modules/Materials.h" #include "modules/Translation.h" @@ -863,7 +864,14 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) if (unit->flags1.bits.ridden || unit->flags4.bits.carrying_somebody) for (auto rider : world->units.other[units_other_id::ANY_RIDER]) if (rider->relationship_ids[unit_relationship_type::RiderMount] == unit->id) - rider->pos = unit->pos; + // Wagon riders need special handling -- they are not at the center of the wagon, + // and the draft animal riders are not even in the extent of the wagon. + if (auto craw = df::creature_raw::find(unit->race); craw && + craw->flags.is_set(df::creature_raw_flags::EQUIPMENT_WAGON)) + Units::teleport(rider, (rider->pos - src_pos) + unit->pos); + else + rider->pos = unit->pos; + return true; } From 937bab6a3a58534c3f2abd65eab912abc224372d Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sat, 19 Sep 2026 23:36:38 -0500 Subject: [PATCH 12/42] templatize `primitive_identity` and subclasses --- docs/dev/data-identity.rst | 18 +- library/DataDefs.cpp | 2 +- library/DataIdentity.cpp | 8 +- library/LuaTypes.cpp | 82 ++------- library/LuaWrapper.cpp | 4 +- library/include/DataIdentity.h | 157 +++++++++++------- .../check-structures-sanity/dispatch.cpp | 10 +- .../devel/check-structures-sanity/types.cpp | 4 +- 8 files changed, 128 insertions(+), 157 deletions(-) diff --git a/docs/dev/data-identity.rst b/docs/dev/data-identity.rst index ae7076941d..46b112cc7a 100644 --- a/docs/dev/data-identity.rst +++ b/docs/dev/data-identity.rst @@ -99,23 +99,23 @@ There are plethora of subclasses of ``type_identity``: * ``function_identity`` (template) wrapper around a C++ function that can be invoked from Lua - * ``primitive_identity`` wrapper around a primitive type. primitive types are fixed-length objects with no internal structure + * ``primitive_identity_base`` abstract base class for primitive types. primitive types are fixed-length objects with no internal structure - * ``bool_identity`` ``bool`` + * ``primitive_identity`` (template) wrapper around a primitive type - * ``number_identity_base`` abstract base for numeric types + * ``bool_identity`` (template) ``bool`` - * ``float_identity_base`` abstract base for floating point types + * ``ptr_string_identity`` (template) C-style (``char *``) string - * ``float_identity`` (template) ``double`` and ``float`` + * ``number_identity_base`` abstract base for numeric types - * ``integer_identity_base`` abstract base for integral types + * ``float_identity`` (template) ``double`` and ``float`` - * ``integer_identity`` (template) ``int8_t``, ``int16_t``, ``int32_t``, ``size_t``, etc. lots of these + * ``integer_identity`` (template) ``int8_t``, ``int16_t``, ``int32_t``, ``size_t``, etc. lots of these - * ``pointer_identity`` any arbitrary C++ pointer (other than ``char*``) + * ``pointer_identity_base`` abstract base class for pointer identities - * ``ptr_string_identity`` C-style (``char *``) string + * ``pointer_identity`` (template) any arbitrary C++ pointer (other than ``char*``) Types marked with "(template)" are C++ template types, all parameterized by a single typename. diff --git a/library/DataDefs.cpp b/library/DataDefs.cpp index 97dc331406..ce11f174ed 100644 --- a/library/DataDefs.cpp +++ b/library/DataDefs.cpp @@ -272,7 +272,7 @@ bool struct_identity::is_equivalent(const struct_identity* other) const return true; } -const std::string pointer_identity::getFullName() const +const std::string pointer_identity_base::getFullName() const { return (target ? target->getFullName() : std::string("void")) + "*"; } diff --git a/library/DataIdentity.cpp b/library/DataIdentity.cpp index 111166d90e..fdf59b983a 100644 --- a/library/DataIdentity.cpp +++ b/library/DataIdentity.cpp @@ -44,12 +44,12 @@ namespace df { FLOAT_IDENTITY_TRAITS(float); FLOAT_IDENTITY_TRAITS(double); - const bool_identity identity_traits::identity; + const bool_identity identity_traits::identity; const stl_string_identity identity_traits::identity; const path_identity identity_traits::identity; - const ptr_string_identity identity_traits::identity; - const ptr_string_identity identity_traits::identity; - const pointer_identity identity_traits::identity; + const ptr_string_identity identity_traits::identity; + const ptr_string_identity identity_traits::identity; + const pointer_identity identity_traits::identity(NULL); const stl_ptr_vector_identity identity_traits >::identity; const stl_bit_vector_identity identity_traits >::identity; const bit_array_identity identity_traits >::identity; diff --git a/library/LuaTypes.cpp b/library/LuaTypes.cpp index 0f990a6b84..7368a9ad5b 100644 --- a/library/LuaTypes.cpp +++ b/library/LuaTypes.cpp @@ -115,64 +115,6 @@ void enum_identity::lua_write(lua_State *state, int fname_idx, void *ptr, int va base_type->lua_write(state, fname_idx, ptr, val_index); } -void df::integer_identity_base::lua_read(lua_State *state, int fname_idx, void *ptr) const -{ - lua_pushinteger(state, read(ptr)); -} - -void df::integer_identity_base::lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const -{ - int is_num = 0; - auto value = lua_tointegerx(state, val_index, &is_num); - if (!is_num) - field_error(state, fname_idx, "integer expected", "write"); - write(ptr, value); -} - -void df::float_identity_base::lua_read(lua_State *state, int fname_idx, void *ptr) const -{ - lua_pushnumber(state, read(ptr)); -} - -void df::float_identity_base::lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const -{ - if (!lua_isnumber(state, val_index)) - field_error(state, fname_idx, "number expected", "write"); - - write(ptr, lua_tonumber(state, val_index)); -} - -void df::bool_identity::lua_read(lua_State *state, int fname_idx, void *ptr) const -{ - lua_pushboolean(state, *(bool*)ptr); -} - -void df::bool_identity::lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const -{ - char *pb = (char*)ptr; - - if (lua_isboolean(state, val_index) || lua_isnil(state, val_index)) - *pb = lua_toboolean(state, val_index); - else if (lua_isnumber(state, val_index)) - *pb = lua_tointeger(state, val_index); - else - field_error(state, fname_idx, "boolean or number expected", "write"); -} - -void df::ptr_string_identity::lua_read(lua_State *state, int fname_idx, void *ptr) const -{ - auto pstr = (char**)ptr; - if (*pstr) - lua_pushstring(state, *pstr); - else - lua_pushnil(state); -} - -void df::ptr_string_identity::lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const -{ - field_error(state, fname_idx, "raw pointer string", "write"); -} - void df::stl_string_identity::lua_read(lua_State *state, int fname_idx, void *ptr) const { auto pstr = (std::string*)ptr; @@ -207,12 +149,12 @@ void df::path_identity::lua_write(lua_State* state, int fname_idx, void* ptr, in *(std::filesystem::path*)ptr = std::filesystem::path(str); } -void df::pointer_identity::lua_read(lua_State *state, int fname_idx, void *ptr, const type_identity *target) +void df::pointer_identity_base::lua_read(lua_State *state, int fname_idx, void *ptr, const type_identity *target) { push_object_internal(state, target, *(void**)ptr); } -void df::pointer_identity::lua_read(lua_State *state, int fname_idx, void *ptr) const +void df::pointer_identity_base::lua_read(lua_State *state, int fname_idx, void *ptr) const { lua_read(state, fname_idx, ptr, target); } @@ -276,7 +218,7 @@ static bool is_null(lua_State *state, int val_index) !lua_touserdata(state, val_index)); } -void df::pointer_identity::lua_write(lua_State *state, int fname_idx, void *ptr, +void df::pointer_identity_base::lua_write(lua_State *state, int fname_idx, void *ptr, const type_identity *target, int val_index) { auto pptr = (void**)ptr; @@ -300,7 +242,7 @@ void df::pointer_identity::lua_write(lua_State *state, int fname_idx, void *ptr, } } -void df::pointer_identity::lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const +void df::pointer_identity_base::lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const { lua_write(state, fname_idx, ptr, target, val_index); } @@ -373,14 +315,14 @@ void ptr_container_identity::lua_item_read(lua_State *state, int fname_idx, void { auto id = (type_identity*)lua_touserdata(state, UPVAL_ITEM_ID); void *pitem = item_pointer(&df::identity_traits::identity, ptr, idx); - df::pointer_identity::lua_read(state, fname_idx, pitem, id); + df::pointer_identity_base::lua_read(state, fname_idx, pitem, id); } void ptr_container_identity::lua_item_write(lua_State *state, int fname_idx, void *ptr, int idx, int val_index) const { auto id = (type_identity*)lua_touserdata(state, UPVAL_ITEM_ID); void *pitem = item_pointer(&df::identity_traits::identity, ptr, idx); - df::pointer_identity::lua_write(state, fname_idx, pitem, id, val_index); + df::pointer_identity_base::lua_write(state, fname_idx, pitem, id, val_index); } bool ptr_container_identity::lua_insert2(lua_State *state, int fname_idx, void *ptr, int idx, int val_index) const @@ -388,7 +330,7 @@ bool ptr_container_identity::lua_insert2(lua_State *state, int fname_idx, void * auto id = (type_identity*)lua_touserdata(state, UPVAL_ITEM_ID); void *pitem = NULL; - df::pointer_identity::lua_write(state, fname_idx, &pitem, id, val_index); + df::pointer_identity_base::lua_write(state, fname_idx, &pitem, id, val_index); return insert(ptr, idx, pitem); } @@ -522,7 +464,7 @@ static void read_field(lua_State *state, const struct_field_info *field, void *p return; case struct_field_info::POINTER: - df::pointer_identity::lua_read(state, 2, ptr, field->type); + df::pointer_identity_base::lua_read(state, 2, ptr, field->type); return; case struct_field_info::CONTAINER: @@ -608,7 +550,7 @@ static void write_field(lua_State *state, const struct_field_info *field, void * return; case struct_field_info::POINTER: - df::pointer_identity::lua_write(state, 2, ptr, field->type, value_idx); + df::pointer_identity_base::lua_write(state, 2, ptr, field->type, value_idx); return; case struct_field_info::STATIC_ARRAY: @@ -1840,12 +1782,12 @@ void LuaWrapper::push_adhoc_pointer(lua_State *state, void *ptr, const type_iden { /* * HACK: relies on - * 1) pointer_identity destructor being no-op + * 1) pointer_identity_base destructor being no-op * 2) lua gc never moving objects in memory */ - void *newobj = lua_newuserdata(state, sizeof(pointer_identity)); - id = new (newobj) pointer_identity(target); + void *newobj = lua_newuserdata(state, sizeof(pointer_identity_base)); + id = new (newobj) pointer_identity_base(target); SaveInTable(state, const_cast(target), &DFHACK_PTR_IDTABLE_TOKEN); lua_pop(state, 1); diff --git a/library/LuaWrapper.cpp b/library/LuaWrapper.cpp index ed0f8f5706..e6f3df0ae3 100644 --- a/library/LuaWrapper.cpp +++ b/library/LuaWrapper.cpp @@ -283,8 +283,8 @@ bool LuaWrapper::is_type_compatible(lua_State *state, const type_identity *type1 { case IDTYPE_POINTER: return is_type_compatible(state, - ((const pointer_identity*)type1)->getTarget(), 0, - ((const pointer_identity*)type2)->getTarget(), 0, + ((const pointer_identity_base*)type1)->getTarget(), 0, + ((const pointer_identity_base*)type2)->getTarget(), 0, exact_equal); break; diff --git a/library/include/DataIdentity.h b/library/include/DataIdentity.h index 85f10229e1..b3c0aeae09 100644 --- a/library/include/DataIdentity.h +++ b/library/include/DataIdentity.h @@ -29,6 +29,7 @@ distribution. #include #include #include +#include #include #include #include @@ -74,13 +75,20 @@ namespace DFHack virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override; }; - class DFHACK_EXPORT primitive_identity : public type_identity { - public: - primitive_identity(size_t size) : type_identity(size) {}; + class DFHACK_EXPORT primitive_identity_base : public type_identity { + protected: + primitive_identity_base(size_t size) : type_identity(size) {}; + public: virtual identity_type type() const override { return IDTYPE_PRIMITIVE; } }; + template + class primitive_identity : public primitive_identity_base { + public: + primitive_identity() : primitive_identity_base(sizeof(T)) {}; + }; + class DFHACK_EXPORT opaque_identity : public constructed_identity { const std::string name; @@ -92,12 +100,12 @@ namespace DFHack virtual identity_type type() const override { return IDTYPE_OPAQUE; } }; - class DFHACK_EXPORT pointer_identity : public primitive_identity { + class DFHACK_EXPORT pointer_identity_base : public primitive_identity_base { const type_identity *target; public: - pointer_identity(const type_identity *target = NULL) - : primitive_identity(sizeof(void*)), target(target) {}; + pointer_identity_base(const type_identity *target = NULL) + : primitive_identity_base(sizeof(void*)), target(target) {}; virtual identity_type type() const override { return IDTYPE_POINTER; } @@ -112,6 +120,15 @@ namespace DFHack virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override; }; + template + requires std::is_pointer_v + class pointer_identity : public pointer_identity_base { + public: + pointer_identity() + : pointer_identity_base(df::identity_traits>::get()) {}; + explicit pointer_identity(const type_identity *target) : pointer_identity_base(target) {}; + }; + class DFHACK_EXPORT container_identity : public constructed_identity { protected: const type_identity *item; @@ -197,91 +214,105 @@ namespace DFHack namespace df { using DFHack::function_identity_base; + using DFHack::primitive_identity_base; using DFHack::primitive_identity; using DFHack::opaque_identity; + using DFHack::pointer_identity_base; using DFHack::pointer_identity; using DFHack::container_identity; using DFHack::ptr_container_identity; using DFHack::bit_container_identity; - class DFHACK_EXPORT number_identity_base : public primitive_identity { + class DFHACK_EXPORT number_identity_base : public primitive_identity_base { const char *name; public: number_identity_base(size_t size, const char *name) - : primitive_identity(size), name(name) {}; + : primitive_identity_base(size), name(name) {}; const std::string getFullName() const override { return name; } + virtual bool isInteger() const { return false; } + virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override = 0; virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override = 0; }; - class DFHACK_EXPORT integer_identity_base : public number_identity_base { - public: - integer_identity_base(size_t size, const char *name) - : number_identity_base(size, name) {} - - virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override; - virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override; - - protected: - virtual int64_t read(void *ptr) const = 0; - virtual void write(void *ptr, int64_t val) const = 0; - }; - - class DFHACK_EXPORT float_identity_base : public number_identity_base { + template + class integer_identity : public number_identity_base { public: - float_identity_base(size_t size, const char *name) - : number_identity_base(size, name) {} + integer_identity(const char *name) : number_identity_base(sizeof(T), name) {} - virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override; - virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override; + virtual bool isInteger() const override { return true; } - protected: - virtual double read(void *ptr) const = 0; - virtual void write(void *ptr, double val) const = 0; + virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override { + lua_pushinteger(state, int64_t(*(T*)ptr)); + } + virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override { + int is_num = 0; + auto value = lua_tointegerx(state, val_index, &is_num); + if (!is_num) + DFHack::LuaWrapper::field_error(state, fname_idx, "integer expected", "write"); + *(T*)ptr = T(value); + } }; template - class integer_identity : public integer_identity_base { + class float_identity : public number_identity_base { public: - integer_identity(const char *name) : integer_identity_base(sizeof(T), name) {} + float_identity(const char *name) : number_identity_base(sizeof(T), name) {} - protected: - virtual int64_t read(void *ptr) const override { return int64_t(*(T*)ptr); } - virtual void write(void *ptr, int64_t val) const override { *(T*)ptr = T(val); } - }; - - template - class float_identity : public float_identity_base { - public: - float_identity(const char *name) : float_identity_base(sizeof(T), name) {} + virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override { + lua_pushnumber(state, double(*(T*)ptr)); + } + virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override { + if (!lua_isnumber(state, val_index)) + DFHack::LuaWrapper::field_error(state, fname_idx, "number expected", "write"); - protected: - virtual double read(void *ptr) const override { return double(*(T*)ptr); } - virtual void write(void *ptr, double val) const override { *(T*)ptr = T(val); } + *(T*)ptr = T(lua_tonumber(state, val_index)); + } }; - class DFHACK_EXPORT bool_identity : public primitive_identity { + template + class bool_identity : public primitive_identity { public: - bool_identity() : primitive_identity(sizeof(bool)) {}; + bool_identity() : primitive_identity() {}; const std::string getFullName() const override { return "bool"; } - virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override; - virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override; + virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override { + lua_pushboolean(state, *(T*)ptr); + } + virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override { + char *pb = (char*)ptr; + + if (lua_isboolean(state, val_index) || lua_isnil(state, val_index)) + *pb = lua_toboolean(state, val_index); + else if (lua_isnumber(state, val_index)) + *pb = lua_tointeger(state, val_index); + else + DFHack::LuaWrapper::field_error(state, fname_idx, "boolean or number expected", "write"); + } }; - class DFHACK_EXPORT ptr_string_identity : public primitive_identity { + template + class ptr_string_identity : public primitive_identity { public: - ptr_string_identity() : primitive_identity(sizeof(char*)) {}; + ptr_string_identity() : primitive_identity() {}; const std::string getFullName() const override { return "char*"; } - virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override; - virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override; + virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override { + auto pstr = *(T*)ptr; + if (pstr) + lua_pushstring(state, pstr); + else + lua_pushnil(state); + } + virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override { + DFHack::LuaWrapper::field_error(state, fname_idx, "raw pointer string", "write"); + } }; class DFHACK_EXPORT stl_string_identity : public DFHack::constructed_identity { @@ -591,7 +622,7 @@ namespace df template<> struct DFHACK_EXPORT identity_traits { \ static const bool is_primitive = true; \ static const category##_identity identity; \ - static const category##_identity_base *get() { return &identity; } \ + static const category##_identity *get() { return &identity; } \ }; #define INTEGER_IDENTITY_TRAITS(type) NUMBER_IDENTITY_TRAITS(integer, type) @@ -648,8 +679,8 @@ namespace df template<> struct DFHACK_EXPORT identity_traits { static const bool is_primitive = true; - static const bool_identity identity; - static const bool_identity *get() { return &identity; } + static const bool_identity identity; + static const bool_identity *get() { return &identity; } }; template<> struct DFHACK_EXPORT identity_traits { @@ -665,20 +696,20 @@ namespace df }; template<> struct DFHACK_EXPORT identity_traits { static const bool is_primitive = true; - static const ptr_string_identity identity; - static const ptr_string_identity *get() { return &identity; } + static const ptr_string_identity identity; + static const ptr_string_identity *get() { return &identity; } }; template<> struct DFHACK_EXPORT identity_traits { static const bool is_primitive = true; - static const ptr_string_identity identity; - static const ptr_string_identity *get() { return &identity; } + static const ptr_string_identity identity; + static const ptr_string_identity *get() { return &identity; } }; template<> struct DFHACK_EXPORT identity_traits { static const bool is_primitive = true; - static const pointer_identity identity; - static const pointer_identity *get() { return &identity; } + static const pointer_identity identity; + static const pointer_identity *get() { return &identity; } }; template<> struct DFHACK_EXPORT identity_traits > { @@ -706,7 +737,7 @@ namespace df template struct identity_traits { static const bool is_primitive = true; - static const pointer_identity *get(); + static const pointer_identity *get(); }; #ifdef BUILD_DFHACK_LIB @@ -791,8 +822,8 @@ namespace df #endif template - inline const pointer_identity *identity_traits::get() { - static const pointer_identity identity(identity_traits::get()); + inline const pointer_identity *identity_traits::get() { + static const pointer_identity identity; return &identity; } diff --git a/plugins/devel/check-structures-sanity/dispatch.cpp b/plugins/devel/check-structures-sanity/dispatch.cpp index a013048e8c..200db9f326 100644 --- a/plugins/devel/check-structures-sanity/dispatch.cpp +++ b/plugins/devel/check-structures-sanity/dispatch.cpp @@ -303,14 +303,12 @@ void Checker::dispatch_primitive(const QueueItem & item, const CheckedStructure FAIL("invalid value for bool: " << int(val)); } } - else if (dynamic_cast(cs.identity)) + else if (auto num_identity = dynamic_cast(cs.identity)) { - check_possible_pointer(item, cs); + if (num_identity->isInteger()) + check_possible_pointer(item, cs); // TODO check ints? - } - else if (dynamic_cast(cs.identity)) - { // TODO check floats? } else @@ -336,7 +334,7 @@ void Checker::dispatch_pointer(const QueueItem & item, const CheckedStructure & } QueueItem target_item(item.path, target_ptr); - auto target = static_cast(cs.identity)->getTarget(); + auto target = static_cast(cs.identity)->getTarget(); if (!target) { check_unknown_pointer(target_item); diff --git a/plugins/devel/check-structures-sanity/types.cpp b/plugins/devel/check-structures-sanity/types.cpp index 4c8a8f6ae2..c0257640c7 100644 --- a/plugins/devel/check-structures-sanity/types.cpp +++ b/plugins/devel/check-structures-sanity/types.cpp @@ -167,13 +167,13 @@ const type_identity *Checker::wrap_in_stl_ptr_vector(const type_identity *base) const type_identity *Checker::wrap_in_pointer(const type_identity *base) { - static std::map> wrappers; + static std::map> wrappers; auto it = wrappers.find(base); if (it != wrappers.end()) { return it->second.get(); } - return (wrappers[base] = std::make_unique(base)).get(); + return (wrappers[base] = std::make_unique(base)).get(); } std::map> known_types_by_size; From c7eef08df101d7fbf9d30a3c38162d24a5e94834 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sun, 20 Sep 2026 00:46:17 -0500 Subject: [PATCH 13/42] collapse some identities combine `integer_identity`, `float_identity`, and `bool_identity` into a single type --- docs/dev/data-identity.rst | 8 +-- library/DataIdentity.cpp | 37 +++++----- library/include/DataIdentity.h | 124 +++++++++++++-------------------- 3 files changed, 69 insertions(+), 100 deletions(-) diff --git a/docs/dev/data-identity.rst b/docs/dev/data-identity.rst index 46b112cc7a..3fd2585244 100644 --- a/docs/dev/data-identity.rst +++ b/docs/dev/data-identity.rst @@ -103,15 +103,11 @@ There are plethora of subclasses of ``type_identity``: * ``primitive_identity`` (template) wrapper around a primitive type - * ``bool_identity`` (template) ``bool`` - * ``ptr_string_identity`` (template) C-style (``char *``) string - * ``number_identity_base`` abstract base for numeric types - - * ``float_identity`` (template) ``double`` and ``float`` + * ``number_identity_base`` abstract base for numeric types (and ``bool``) - * ``integer_identity`` (template) ``int8_t``, ``int16_t``, ``int32_t``, ``size_t``, etc. lots of these + * ``number_identity`` (template) ``bool``, ``int8_t``, ``int16_t``, ``int32_t``, ``size_t``, ``float``, ``double``, etc. lots of these * ``pointer_identity_base`` abstract base class for pointer identities diff --git a/library/DataIdentity.cpp b/library/DataIdentity.cpp index fdf59b983a..f3b27eadd5 100644 --- a/library/DataIdentity.cpp +++ b/library/DataIdentity.cpp @@ -21,30 +21,27 @@ // without it the macro generates a syntax error when type is a template specification namespace df { -#define NUMBER_IDENTITY_TRAITS(category, type, name) \ - const category##_identity identity_traits::identity(name); -#define INTEGER_IDENTITY_TRAITS(type, name) NUMBER_IDENTITY_TRAITS(integer, type, name) -#define FLOAT_IDENTITY_TRAITS(type) NUMBER_IDENTITY_TRAITS(float, type, #type) +#define NUMBER_IDENTITY_TRAITS(type, name) \ + const number_identity identity_traits::identity(name); #define OPAQUE_IDENTITY_TRAITS_NAME(name, ...) \ const opaque_identity identity_traits<__VA_ARGS__ >::identity(sizeof(__VA_ARGS__), allocator_fn<__VA_ARGS__ >, name) #define OPAQUE_IDENTITY_TRAITS(...) OPAQUE_IDENTITY_TRAITS_NAME(#__VA_ARGS__, __VA_ARGS__ ) - INTEGER_IDENTITY_TRAITS(char, "char"); - INTEGER_IDENTITY_TRAITS(signed char, "int8_t"); - INTEGER_IDENTITY_TRAITS(unsigned char, "uint8_t"); - INTEGER_IDENTITY_TRAITS(short, "int16_t"); - INTEGER_IDENTITY_TRAITS(unsigned short, "uint16_t"); - INTEGER_IDENTITY_TRAITS(int, "int32_t"); - INTEGER_IDENTITY_TRAITS(unsigned int, "uint32_t"); - INTEGER_IDENTITY_TRAITS(long, "long"); - INTEGER_IDENTITY_TRAITS(unsigned long, "unsigned long"); - INTEGER_IDENTITY_TRAITS(long long, "int64_t"); - INTEGER_IDENTITY_TRAITS(unsigned long long, "uint64_t"); - INTEGER_IDENTITY_TRAITS(wchar_t, "wchar_t"); - FLOAT_IDENTITY_TRAITS(float); - FLOAT_IDENTITY_TRAITS(double); - - const bool_identity identity_traits::identity; + NUMBER_IDENTITY_TRAITS(char, "char"); + NUMBER_IDENTITY_TRAITS(signed char, "int8_t"); + NUMBER_IDENTITY_TRAITS(unsigned char, "uint8_t"); + NUMBER_IDENTITY_TRAITS(short, "int16_t"); + NUMBER_IDENTITY_TRAITS(unsigned short, "uint16_t"); + NUMBER_IDENTITY_TRAITS(int, "int32_t"); + NUMBER_IDENTITY_TRAITS(unsigned int, "uint32_t"); + NUMBER_IDENTITY_TRAITS(long, "long"); + NUMBER_IDENTITY_TRAITS(unsigned long, "unsigned long"); + NUMBER_IDENTITY_TRAITS(long long, "int64_t"); + NUMBER_IDENTITY_TRAITS(unsigned long long, "uint64_t"); + NUMBER_IDENTITY_TRAITS(wchar_t, "wchar_t"); + NUMBER_IDENTITY_TRAITS(float, "float"); + NUMBER_IDENTITY_TRAITS(double, "double"); + NUMBER_IDENTITY_TRAITS(bool, "bool"); const stl_string_identity identity_traits::identity; const path_identity identity_traits::identity; const ptr_string_identity identity_traits::identity; diff --git a/library/include/DataIdentity.h b/library/include/DataIdentity.h index b3c0aeae09..1a6fe61688 100644 --- a/library/include/DataIdentity.h +++ b/library/include/DataIdentity.h @@ -240,59 +240,45 @@ namespace df }; template - class integer_identity : public number_identity_base { + class number_identity : public number_identity_base { public: - integer_identity(const char *name) : number_identity_base(sizeof(T), name) {} + number_identity(const char *name) : number_identity_base(sizeof(T), name) {} - virtual bool isInteger() const override { return true; } - - virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override { - lua_pushinteger(state, int64_t(*(T*)ptr)); - } - virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override { - int is_num = 0; - auto value = lua_tointegerx(state, val_index, &is_num); - if (!is_num) - DFHack::LuaWrapper::field_error(state, fname_idx, "integer expected", "write"); - *(T*)ptr = T(value); + virtual bool isInteger() const override { + return std::is_integral_v && !std::is_same_v; } - }; - - template - class float_identity : public number_identity_base { - public: - float_identity(const char *name) : number_identity_base(sizeof(T), name) {} virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override { - lua_pushnumber(state, double(*(T*)ptr)); - } - virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override { - if (!lua_isnumber(state, val_index)) - DFHack::LuaWrapper::field_error(state, fname_idx, "number expected", "write"); - - *(T*)ptr = T(lua_tonumber(state, val_index)); - } - }; - - template - class bool_identity : public primitive_identity { - public: - bool_identity() : primitive_identity() {}; - - const std::string getFullName() const override { return "bool"; } - - virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override { - lua_pushboolean(state, *(T*)ptr); + if constexpr (std::is_same_v) { + lua_pushboolean(state, *(T*)ptr); + } else if constexpr (std::is_floating_point_v) { + lua_pushnumber(state, double(*(T*)ptr)); + } else { + lua_pushinteger(state, int64_t(*(T*)ptr)); + } } virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override { - char *pb = (char*)ptr; - - if (lua_isboolean(state, val_index) || lua_isnil(state, val_index)) - *pb = lua_toboolean(state, val_index); - else if (lua_isnumber(state, val_index)) - *pb = lua_tointeger(state, val_index); - else - DFHack::LuaWrapper::field_error(state, fname_idx, "boolean or number expected", "write"); + if constexpr (std::is_same_v) { + char *pb = (char*)ptr; + + if (lua_isboolean(state, val_index) || lua_isnil(state, val_index)) + *pb = lua_toboolean(state, val_index); + else if (lua_isnumber(state, val_index)) + *pb = lua_tointeger(state, val_index); + else + DFHack::LuaWrapper::field_error(state, fname_idx, "boolean or number expected", "write"); + } else if constexpr (std::is_floating_point_v) { + if (!lua_isnumber(state, val_index)) + DFHack::LuaWrapper::field_error(state, fname_idx, "number expected", "write"); + + *(T*)ptr = T(lua_tonumber(state, val_index)); + } else { + int is_num = 0; + auto value = lua_tointegerx(state, val_index, &is_num); + if (!is_num) + DFHack::LuaWrapper::field_error(state, fname_idx, "integer expected", "write"); + *(T*)ptr = T(value); + } } }; @@ -618,16 +604,13 @@ namespace df }; #endif -#define NUMBER_IDENTITY_TRAITS(category, type) \ +#define NUMBER_IDENTITY_TRAITS(type) \ template<> struct DFHACK_EXPORT identity_traits { \ static const bool is_primitive = true; \ - static const category##_identity identity; \ - static const category##_identity *get() { return &identity; } \ + static const number_identity identity; \ + static const number_identity *get() { return &identity; } \ }; -#define INTEGER_IDENTITY_TRAITS(type) NUMBER_IDENTITY_TRAITS(integer, type) -#define FLOAT_IDENTITY_TRAITS(type) NUMBER_IDENTITY_TRAITS(float, type) - // the space after the use of "type" in OPAQUE_IDENTITY_TRAITS is _required_ // without it the macro generates a syntax error when type is a template specification @@ -637,20 +620,21 @@ namespace df static const opaque_identity *get() { return &identity; } \ }; - INTEGER_IDENTITY_TRAITS(char); - INTEGER_IDENTITY_TRAITS(signed char); - INTEGER_IDENTITY_TRAITS(unsigned char); - INTEGER_IDENTITY_TRAITS(short); - INTEGER_IDENTITY_TRAITS(unsigned short); - INTEGER_IDENTITY_TRAITS(int); - INTEGER_IDENTITY_TRAITS(unsigned int); - INTEGER_IDENTITY_TRAITS(long); - INTEGER_IDENTITY_TRAITS(unsigned long); - INTEGER_IDENTITY_TRAITS(long long); - INTEGER_IDENTITY_TRAITS(unsigned long long); - INTEGER_IDENTITY_TRAITS(wchar_t); - FLOAT_IDENTITY_TRAITS(float); - FLOAT_IDENTITY_TRAITS(double); + NUMBER_IDENTITY_TRAITS(char); + NUMBER_IDENTITY_TRAITS(signed char); + NUMBER_IDENTITY_TRAITS(unsigned char); + NUMBER_IDENTITY_TRAITS(short); + NUMBER_IDENTITY_TRAITS(unsigned short); + NUMBER_IDENTITY_TRAITS(int); + NUMBER_IDENTITY_TRAITS(unsigned int); + NUMBER_IDENTITY_TRAITS(long); + NUMBER_IDENTITY_TRAITS(unsigned long); + NUMBER_IDENTITY_TRAITS(long long); + NUMBER_IDENTITY_TRAITS(unsigned long long); + NUMBER_IDENTITY_TRAITS(wchar_t); + NUMBER_IDENTITY_TRAITS(float); + NUMBER_IDENTITY_TRAITS(double); + NUMBER_IDENTITY_TRAITS(bool); OPAQUE_IDENTITY_TRAITS(wchar_t*); OPAQUE_IDENTITY_TRAITS(std::condition_variable); OPAQUE_IDENTITY_TRAITS(std::fstream); @@ -677,12 +661,6 @@ namespace df }; #endif - template<> struct DFHACK_EXPORT identity_traits { - static const bool is_primitive = true; - static const bool_identity identity; - static const bool_identity *get() { return &identity; } - }; - template<> struct DFHACK_EXPORT identity_traits { static const bool is_primitive = true; static const stl_string_identity identity; @@ -723,8 +701,6 @@ namespace df }; #undef NUMBER_IDENTITY_TRAITS -#undef INTEGER_IDENTITY_TRAITS -#undef FLOAT_IDENTITY_TRAITS #undef OPAQUE_IDENTITY_TRAITS // Container declarations From 0047aee34816b26bb6408aa6624fbfde6b7326ee Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sun, 20 Sep 2026 03:07:02 -0500 Subject: [PATCH 14/42] refactor type identities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `type_identity` (non-template, `DataDefs.h`) — the erased abstract base; all virtuals (`type`, `getFullName`, `isPrimitive`, `lua_read`/`lua_write`, `build_metatable`, `allocate`/`copy`/`destroy`) live here `type_identity_for` (`DataIdentity.h`) — the single leaf template. Its base is chosen by `identity_base_of_t`: `T::df_identity_base` typedef wins outright (codegen emits it into every compound type) `is_enum` / `enum_field` → `enum_identity`; `bitfield_traits` → `bitfield_identity` types with `_identity` → `struct_identity` / `union_identity` / `virtual_identity` / `other_vectors_identity` via `is_union`/`is_polymorphic`/marker-typedef checks `char*`/`const char*` → `primitive_identity_base`; `wchar_t*` → `opaque_identity`; other pointers → `pointer_identity_base`; arithmetic + bool → `number_identity_base` `std::string/std::filesystem::path` → `constructed_identity`; C arrays/`std::array` → `buffer_container_identity`; STL/`BitArray`/`DfArray`/`enum_list_attr` → `detail::container_impl`; other classes → `opaque_identity` `lua_read`/`lua_write`/`type()`/`getFullName()`/`isPrimitive()`/`build_metatable`/`isInteger()` are implemented once with `if constexpr` on `T`. `global_identity` is `type_identity` over an empty placeholder with `df_identity_base` = `struct_identity`. --- docs/dev/data-identity.rst | 55 +- library/DataIdentity.cpp | 24 +- library/LuaTypes.cpp | 24 +- library/LuaWrapper.cpp | 4 +- library/include/BitArray.h | 2 + library/include/DataDefs.h | 27 +- library/include/DataIdentity.h | 856 ++++++++++-------- library/include/LuaTools.h | 1 + library/xml | 2 +- .../devel/check-structures-sanity/types.cpp | 4 +- plugins/stonesense | 2 +- scripts | 2 +- 12 files changed, 559 insertions(+), 444 deletions(-) diff --git a/docs/dev/data-identity.rst b/docs/dev/data-identity.rst index 3fd2585244..2300242ff9 100644 --- a/docs/dev/data-identity.rst +++ b/docs/dev/data-identity.rst @@ -17,6 +17,13 @@ contains both a pointer to the C++ object itself and a pointer to a ``type_ident by that pointer. Note that the userdata object does not own the objects pointed to by these pointers, and the Lua engine is never responsible for managing their lifetimes. +Most identities are instances of the single template class ``type_identity_for``, which wraps the C++ type ``T``. +``type_identity_for`` derives from a runtime base class selected at compile time by inspection of ``T`` +(via ``std::is_enum``, ``std::is_pointer``, ``std::is_arithmetic``, container traits, and similar checks), +and implements the applicable virtual methods with ``if constexpr`` dispatch on the properties of ``T``. +A type can override the selection by defining a ``df_identity_base`` typedef naming the desired base; +the code generator emits this typedef for all generated compound types. + ``type_identity`` defines the following public methods: - ``byte_size``: returns the size, in bytes, of the object held @@ -33,7 +40,7 @@ never responsible for managing their lifetimes. - ``is_primitive``: indicates that ``lua_read`` will store a *copy* of the object on the Lua stack instead of a non-owning reference to it. Used for types that have direct representations in Lua: numbers, booleans, simple strings -- ``is_constructed``: indicates that creating a C++ instance of this type requires the use of a possibly nontrivial constructor. A type identity that is both primitive and constructed cannot be inserted into a container. At the moment the only type identity that is both primitive and constructed is ``stl_string_identity``, which wraps the C++ ``std::string`` type. +- ``is_constructed``: indicates that creating a C++ instance of this type requires the use of a possibly nontrivial constructor. A type identity that is both primitive and constructed cannot be inserted into a container. At the moment the only type identity that is both primitive and constructed is ``type_identity_for``, which wraps the C++ ``std::string`` type. - ``is_container``: indicates that the type is a container and thus implements the methods specific to ``container_identity`` @@ -41,7 +48,9 @@ never responsible for managing their lifetimes. - ``copy``: copy the object at ``src`` onto ``tgt``. This uses ``memmove`` for primitive types, and C++ copy-assignment (when possible) for other types -There are plethora of subclasses of ``type_identity``: +The identity class tree is divided into two layers: the *runtime base classes*, which are ordinary +(non-template) classes used for polymorphic dispatch, and the single leaf template ``type_identity_for``, +which derives from the appropriate runtime base for the C++ type ``T`` that it wraps: * ``type_identity`` the abstract base class of all type identities @@ -55,65 +64,45 @@ There are plethora of subclasses of ``type_identity``: * ``struct_identity`` C++ ``class`` or ``structure`` - * ``global_identity`` holds, as a quasiobject, handles for all of the known Dwarf Fortress program-scope static objects as if they were fields of an object called ``global`` - * ``union_identity`` C++ ``union`` * ``other_vectors_identity`` special-case identity for the categorized subvectors of objects that appears in many of Dwarf Fortress's "handler" classes * ``virtual_identity`` polymorphic C++ ``class`` or ``structure`` having a virtual table to handle virtual dispatch - * ``stl_string_identity`` ``std::string`` - * ``xlsx_file_handle_identity`` special case * ``xlsx_sheet_handle_identity`` special case * ``container_identity`` "containers" generally. note that all container types are homogeneous (that is, the elements of the container must all be of the same type). abstract base class - * ``bit_container_identity`` for containers that contain bools stored one element per *bit* (rather than per byte) - - * ``bit_array_identity`` Dwarf Fortress's ``BitArray`` type - - * ``stl_bit_vector_identity`` ``std::vector`` + * ``bit_container_identity`` for containers that contain bools stored one element per *bit* (rather than per byte); also provides the ``get_item``/``set_item`` virtual interface * ``buffer_container_identity`` C++ static arrays and raw C++ pointers acting as arrays of unspecified bound - * ``enum_list_attr_identity`` (template) metaobject with metadata about a C++ enumeration; may also include additional metadata - * ``ptr_container_identity`` containers that contain pointers - * ``stl_ptr_container_identity`` containers that are of the form ``std::vector`` for some ``T`` - - * ``ro_stl_container_identity`` (template) "read only containers" - - * ``ro_stl_assoc_container_identity`` (template) ``std::map`` and ``std::unordered_map`` - - * ``stl_container_identity`` (template) ``std::vector`` where ``T`` is *not* a pointer (and not ``bool``) - * ``opaque_identity`` opaque wrapper around any type, provides no functionality - * ``stl_string_identity`` ``std::string`` - * ``function_identity_base`` abstract base class for ``function_identity`` * ``function_identity`` (template) wrapper around a C++ function that can be invoked from Lua * ``primitive_identity_base`` abstract base class for primitive types. primitive types are fixed-length objects with no internal structure - * ``primitive_identity`` (template) wrapper around a primitive type - - * ``ptr_string_identity`` (template) C-style (``char *``) string - - * ``number_identity_base`` abstract base for numeric types (and ``bool``) - - * ``number_identity`` (template) ``bool``, ``int8_t``, ``int16_t``, ``int32_t``, ``size_t``, ``float``, ``double``, etc. lots of these + * ``number_identity_base`` abstract base for numeric types (and ``bool``); provides the ``isInteger`` discriminator - * ``pointer_identity_base`` abstract base class for pointer identities + * ``pointer_identity_base`` abstract base class for pointer identities; provides ``getTarget`` - * ``pointer_identity`` (template) any arbitrary C++ pointer (other than ``char*``) +* ``type_identity_for`` the leaf template that provides the identity of the C++ type ``T``. Its runtime + base class is selected by compile-time inspection of ``T`` (or by a ``df_identity_base`` typedef in ``T``), + covering all of the categories above: enums, bitfields, structs, unions, virtual classes, numbers, + pointers, C strings, ``std::string``, ``std::filesystem::path``, static arrays, STL containers, + ``BitArray``, ``DfArray``, ``enum_list_attr``, ``std::vector``, bit containers, and opaque types. -Types marked with "(template)" are C++ template types, all parameterized by a single typename. + ``global_identity`` is an alias for ``type_identity_for``, where ``global_object`` is an + empty placeholder type: it holds, as a quasiobject, handles for all of the known Dwarf Fortress + program-scope static objects as if they were fields of an object called ``global``. Type identity object lifetime and mutability ============================================ diff --git a/library/DataIdentity.cpp b/library/DataIdentity.cpp index f3b27eadd5..cc6ea64fa6 100644 --- a/library/DataIdentity.cpp +++ b/library/DataIdentity.cpp @@ -22,9 +22,9 @@ namespace df { #define NUMBER_IDENTITY_TRAITS(type, name) \ - const number_identity identity_traits::identity(name); + const type_identity_for identity_traits::identity(name); #define OPAQUE_IDENTITY_TRAITS_NAME(name, ...) \ - const opaque_identity identity_traits<__VA_ARGS__ >::identity(sizeof(__VA_ARGS__), allocator_fn<__VA_ARGS__ >, name) + const type_identity_for<__VA_ARGS__ > identity_traits<__VA_ARGS__ >::identity(sizeof(__VA_ARGS__), allocator_fn<__VA_ARGS__ >, name) #define OPAQUE_IDENTITY_TRAITS(...) OPAQUE_IDENTITY_TRAITS_NAME(#__VA_ARGS__, __VA_ARGS__ ) NUMBER_IDENTITY_TRAITS(char, "char"); @@ -42,14 +42,14 @@ namespace df { NUMBER_IDENTITY_TRAITS(float, "float"); NUMBER_IDENTITY_TRAITS(double, "double"); NUMBER_IDENTITY_TRAITS(bool, "bool"); - const stl_string_identity identity_traits::identity; - const path_identity identity_traits::identity; - const ptr_string_identity identity_traits::identity; - const ptr_string_identity identity_traits::identity; - const pointer_identity identity_traits::identity(NULL); - const stl_ptr_vector_identity identity_traits >::identity; - const stl_bit_vector_identity identity_traits >::identity; - const bit_array_identity identity_traits >::identity; + const type_identity_for identity_traits::identity; + const type_identity_for identity_traits::identity; + const type_identity_for identity_traits::identity; + const type_identity_for identity_traits::identity; + const type_identity_for identity_traits::identity(nullptr); + const type_identity_for > identity_traits >::identity; + const type_identity_for > identity_traits >::identity; + const type_identity_for > identity_traits >::identity; OPAQUE_IDENTITY_TRAITS(std::condition_variable); OPAQUE_IDENTITY_TRAITS(std::fstream); @@ -67,6 +67,6 @@ namespace df { const buffer_container_identity buffer_container_identity::base_instance; - const stl_container_identity > stl_vector_int32_t_identity("vector", identity_traits::get()); - const stl_container_identity > stl_vector_int16_t_identity("vector", identity_traits::get()); + const type_identity_for > stl_vector_int32_t_identity("vector", identity_traits::get()); + const type_identity_for > stl_vector_int16_t_identity("vector", identity_traits::get()); } diff --git a/library/LuaTypes.cpp b/library/LuaTypes.cpp index 7368a9ad5b..c9922689ca 100644 --- a/library/LuaTypes.cpp +++ b/library/LuaTypes.cpp @@ -115,30 +115,14 @@ void enum_identity::lua_write(lua_State *state, int fname_idx, void *ptr, int va base_type->lua_write(state, fname_idx, ptr, val_index); } -void df::stl_string_identity::lua_read(lua_State *state, int fname_idx, void *ptr) const -{ - auto pstr = (std::string*)ptr; - lua_pushlstring(state, pstr->data(), pstr->size()); -} - -void df::stl_string_identity::lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const -{ - size_t size; - const char *bytes = lua_tolstring(state, val_index, &size); - if (!bytes) - field_error(state, fname_idx, "string expected", "write"); - - *(std::string*)ptr = std::string(bytes, size); -} - -void df::path_identity::lua_read(lua_State* state, int fname_idx, void* ptr) const +void DFHack::lua_read_path(lua_State* state, void* ptr) { auto ppath = (std::filesystem::path*)ptr; auto str = DFHack::Filesystem::as_string(*ppath); lua_pushlstring(state, (char*)str.data(), str.size()); } -void df::path_identity::lua_write(lua_State* state, int fname_idx, void* ptr, int val_index) const +void DFHack::lua_write_path(lua_State* state, int fname_idx, void* ptr, int val_index) { size_t size; const char* bytes = lua_tolstring(state, val_index, &size); @@ -1713,10 +1697,10 @@ void other_vectors_identity::build_metatable(lua_State *state) const SetPtrMethods(state, base+1, base+2); } -void global_identity::build_metatable(lua_State *state) const +void DFHack::build_global_metatable(lua_State *state, const struct_identity *id) { int base = lua_gettop(state); - MakeFieldMetatable(state, this, meta_global_index, meta_global_newindex, meta_struct_next, true); + MakeFieldMetatable(state, id, meta_global_index, meta_global_newindex, meta_struct_next, true); SetStructMethod(state, base+1, base+2, meta_global_field_reference, "_field"); SetPtrMethods(state, base+1, base+2); } diff --git a/library/LuaWrapper.cpp b/library/LuaWrapper.cpp index e6f3df0ae3..0d30429297 100644 --- a/library/LuaWrapper.cpp +++ b/library/LuaWrapper.cpp @@ -304,8 +304,8 @@ bool LuaWrapper::is_type_compatible(lua_State *state, const type_identity *type1 case IDTYPE_STL_PTR_VECTOR: { - auto b1 = (const df::stl_ptr_vector_identity*)type1; - auto b2 = (const df::stl_ptr_vector_identity*)type2; + auto b1 = (const df::ptr_container_identity*)type1; + auto b2 = (const df::ptr_container_identity*)type2; const type_identity *item1 = b1->getItemType(), *item2 = b2->getItemType(); fetch_container_details(state, meta1, &item1, NULL); diff --git a/library/include/BitArray.h b/library/include/BitArray.h index 7658e33102..3de148c57a 100644 --- a/library/include/BitArray.h +++ b/library/include/BitArray.h @@ -609,6 +609,8 @@ namespace DFHack template struct DfOtherVectors { + using dfhack_other_vectors = void; + std::vector & operator[](O other_id) { CHECK_INVALID_ARGUMENT(size_t(other_id) < sizeof(T) / sizeof(std::vector)); diff --git a/library/include/DataDefs.h b/library/include/DataDefs.h index dd910b3a8e..8ff99c750c 100644 --- a/library/include/DataDefs.h +++ b/library/include/DataDefs.h @@ -77,6 +77,8 @@ namespace DFHack using TAllocateFn = void *(*)(void*, const void*); + template class type_identity_for; + class DFHACK_EXPORT type_identity { const size_t size; @@ -327,16 +329,14 @@ namespace DFHack bool is_equivalent(const struct_identity* other) const; }; - class DFHACK_EXPORT global_identity : public struct_identity { - public: - global_identity(const struct_field_info *fields) - : struct_identity(0,NULL,NULL,"global",NULL,fields) {} - - virtual identity_type type() const override { return IDTYPE_GLOBAL; } - - virtual void build_metatable(lua_State *state) const override; + // Placeholder type wrapped by the identity of the global "object", + // which does not correspond to an actual C++ type. + struct global_object { + using df_identity_base = struct_identity; }; + using global_identity = type_identity_for; + class DFHACK_EXPORT union_identity : public struct_identity { public: union_identity(size_t size, TAllocateFn alloc, @@ -516,6 +516,7 @@ inline int linear_index(const DFHack::enum_list_attr &lst, const st namespace df { + using DFHack::type_identity_for; using DFHack::type_identity; using DFHack::compound_identity; using DFHack::virtual_ptr; @@ -603,8 +604,16 @@ namespace df template struct identity_traits {}; + /* + * Compound types (structs, unions, classes) are recognized either by an + * explicit df_identity_base typedef that derives from compound_identity + * (emitted by the code generator), or by the _identity member itself + * being convertible to a compound identity. + */ template - requires requires () { { &T::_identity } -> std::convertible_to; } + requires (requires { typename T::df_identity_base; } && + std::is_base_of_v) || + requires () { { &T::_identity } -> std::convertible_to; } struct identity_traits { static const bool is_primitive = false; static const compound_identity *get() { return &T::_identity; } diff --git a/library/include/DataIdentity.h b/library/include/DataIdentity.h index 1a6fe61688..0bad6c4993 100644 --- a/library/include/DataIdentity.h +++ b/library/include/DataIdentity.h @@ -24,8 +24,10 @@ distribution. #pragma once +#include #include #include +#include #include #include #include @@ -83,12 +85,6 @@ namespace DFHack virtual identity_type type() const override { return IDTYPE_PRIMITIVE; } }; - template - class primitive_identity : public primitive_identity_base { - public: - primitive_identity() : primitive_identity_base(sizeof(T)) {}; - }; - class DFHACK_EXPORT opaque_identity : public constructed_identity { const std::string name; @@ -120,15 +116,6 @@ namespace DFHack virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override; }; - template - requires std::is_pointer_v - class pointer_identity : public pointer_identity_base { - public: - pointer_identity() - : pointer_identity_base(df::identity_traits>::get()) {}; - explicit pointer_identity(const type_identity *target) : pointer_identity_base(target) {}; - }; - class DFHACK_EXPORT container_identity : public constructed_identity { protected: const type_identity *item; @@ -215,10 +202,8 @@ namespace df { using DFHack::function_identity_base; using DFHack::primitive_identity_base; - using DFHack::primitive_identity; using DFHack::opaque_identity; using DFHack::pointer_identity_base; - using DFHack::pointer_identity; using DFHack::container_identity; using DFHack::ptr_container_identity; using DFHack::bit_container_identity; @@ -233,153 +218,13 @@ namespace df const std::string getFullName() const override { return name; } virtual bool isInteger() const { return false; } - - virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override = 0; - virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override = 0; - }; - template - class number_identity : public number_identity_base { - public: - number_identity(const char *name) : number_identity_base(sizeof(T), name) {} - - virtual bool isInteger() const override { - return std::is_integral_v && !std::is_same_v; - } - - virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override { - if constexpr (std::is_same_v) { - lua_pushboolean(state, *(T*)ptr); - } else if constexpr (std::is_floating_point_v) { - lua_pushnumber(state, double(*(T*)ptr)); - } else { - lua_pushinteger(state, int64_t(*(T*)ptr)); - } - } - virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override { - if constexpr (std::is_same_v) { - char *pb = (char*)ptr; - - if (lua_isboolean(state, val_index) || lua_isnil(state, val_index)) - *pb = lua_toboolean(state, val_index); - else if (lua_isnumber(state, val_index)) - *pb = lua_tointeger(state, val_index); - else - DFHack::LuaWrapper::field_error(state, fname_idx, "boolean or number expected", "write"); - } else if constexpr (std::is_floating_point_v) { - if (!lua_isnumber(state, val_index)) - DFHack::LuaWrapper::field_error(state, fname_idx, "number expected", "write"); - - *(T*)ptr = T(lua_tonumber(state, val_index)); - } else { - int is_num = 0; - auto value = lua_tointegerx(state, val_index, &is_num); - if (!is_num) - DFHack::LuaWrapper::field_error(state, fname_idx, "integer expected", "write"); - *(T*)ptr = T(value); - } - } - }; - - template - class ptr_string_identity : public primitive_identity { - public: - ptr_string_identity() : primitive_identity() {}; - - const std::string getFullName() const override { return "char*"; } - - virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override { - auto pstr = *(T*)ptr; - if (pstr) - lua_pushstring(state, pstr); - else - lua_pushnil(state); - } - virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override { - DFHack::LuaWrapper::field_error(state, fname_idx, "raw pointer string", "write"); - } - }; - - class DFHACK_EXPORT stl_string_identity : public DFHack::constructed_identity { - public: - stl_string_identity() - : constructed_identity(sizeof(std::string), &allocator_fn) - {}; - - const std::string getFullName() const override { return "string"; } - - virtual DFHack::identity_type type() const override { return DFHack::IDTYPE_PRIMITIVE; } - - virtual bool isPrimitive() const override { return true; } - - virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override; - virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override; - }; - - class DFHACK_EXPORT path_identity : public DFHack::constructed_identity { - public: - path_identity() - : constructed_identity(sizeof(std::filesystem::path), &allocator_fn) - { - }; - - const std::string getFullName() const override { return "path"; } - - virtual DFHack::identity_type type() const override { return DFHack::IDTYPE_PRIMITIVE; } - - virtual bool isPrimitive() const override { return true; } - - virtual void lua_read(lua_State* state, int fname_idx, void* ptr) const override; - virtual void lua_write(lua_State* state, int fname_idx, void* ptr, int val_index) const override; - }; - - - class DFHACK_EXPORT stl_ptr_vector_identity : public ptr_container_identity { - public: - using container = std::vector; - - /* - * This class assumes that std::vector is equivalent - * in layout and behavior to std::vector for any T. - */ - - stl_ptr_vector_identity(const type_identity *item = NULL, const enum_identity *ienum = NULL) - : ptr_container_identity(sizeof(container), &df::allocator_fn, item, ienum) - {}; - - const std::string getFullName(const type_identity *item) const override { - return "vector" + ptr_container_identity::getFullName(item); - } - - virtual DFHack::identity_type type() const override { return DFHack::IDTYPE_STL_PTR_VECTOR; } - - virtual bool resize(void *ptr, int size) const override { - (*(container*)ptr).resize(size); - return true; - } - virtual bool erase(void *ptr, int size) const override { - auto &ct = *(container*)ptr; - ct.erase(ct.begin()+size); - return true; - } - virtual bool insert(void *ptr, int idx, void *item) const override { - auto &ct = *(container*)ptr; - ct.insert(ct.begin()+idx, item); - return true; - } - - protected: - virtual int item_count(void *ptr, CountMode) const override { - return (int)((container*)ptr)->size(); - }; - virtual void *item_pointer(const type_identity *, void *ptr, int idx) const override { - return &(*(container*)ptr)[idx]; - } - }; - -// Due to export issues, this stuff can only work in the main dll -#ifdef BUILD_DFHACK_LIB + /* + * Identity for statically-sized arrays (C arrays and std::array). + * Unlike other container identities the item count is fixed, + * so instances can also serve as the identity of ad-hoc buffers. + */ class buffer_container_identity : public container_identity { int size; @@ -394,6 +239,7 @@ namespace df size_t byte_size() const override { return getItemType()->byte_size()*size; } + using container_identity::getFullName; const std::string getFullName(const type_identity *item) const override; int getSize() const { return size; } @@ -407,208 +253,499 @@ namespace df return ((uint8_t*)ptr) + idx * item->byte_size(); } }; -#endif +} - template - class stl_container_identity : public container_identity { - const char *name; +namespace DFHack +{ + namespace detail + { + /* + * Compile-time type inspection used by type_identity_for to pick + * the appropriate behavior for the wrapped C++ type. + */ - public: - stl_container_identity(const char *name, const type_identity *item, const enum_identity *ienum = NULL) - : container_identity(sizeof(T), &allocator_fn, item, ienum), name(name) - {} + template struct is_std_vector : std::false_type {}; + template struct is_std_vector> : std::true_type {}; - const std::string getFullName(const type_identity *item) const override { - return name + container_identity::getFullName(item); - } + template struct is_std_deque : std::false_type {}; + template struct is_std_deque> : std::true_type {}; - virtual bool resize(void *ptr, int size) const override { - (*(T*)ptr).resize(size); - return true; - } - virtual bool erase(void *ptr, int size) const override { - auto &ct = *(T*)ptr; - ct.erase(ct.begin()+size); - return true; - } - virtual bool insert(void *ptr, int idx, void *item) const override { - auto &ct = *(T*)ptr; - ct.insert(ct.begin()+idx, *(typename T::value_type*)item); - return true; - } - virtual bool lua_insert2(lua_State* state, int fname_idx, void* ptr, int idx, int val_index) const override - { - using VT = typename T::value_type; - VT tmp{}; - auto id = (type_identity*)lua_touserdata(state, DFHack::LuaWrapper::UPVAL_ITEM_ID); - auto pitem = DFHack::LuaWrapper::get_object_internal(state, id, val_index, false); - bool useTemporary = (!pitem && id->isPrimitive()); - - if (useTemporary) - { - pitem = &tmp; - id->lua_write(state, fname_idx, pitem, val_index); - } + template struct is_std_array : std::false_type {}; + template struct is_std_array> : std::true_type {}; - if (id != item || !pitem) - DFHack::LuaWrapper::field_error(state, fname_idx, "incompatible object type", "insert"); + template struct is_std_set : std::false_type {}; + template struct is_std_set> : std::true_type {}; + template struct is_std_set> : std::true_type {}; - return insert(ptr, idx, pitem); - } + template struct is_std_map : std::false_type {}; + template struct is_std_map> : std::true_type {}; + template struct is_std_map> : std::true_type {}; - protected: - virtual int item_count(void *ptr, CountMode) const override { return (int)((T*)ptr)->size(); } - virtual void *item_pointer(const type_identity *item, void *ptr, int idx) const override { - return &(*(T*)ptr)[idx]; - } - }; + template struct is_bit_array : std::false_type {}; + template struct is_bit_array> : std::true_type {}; -#ifdef BUILD_DFHACK_LIB - template - class ro_stl_container_identity : public container_identity { - protected: - const char *name; + template struct is_df_array : std::false_type {}; + template struct is_df_array> : std::true_type {}; - public: - ro_stl_container_identity(const char *name, const type_identity *item, const enum_identity *ienum = NULL) - : container_identity(sizeof(T), &allocator_fn, item, ienum), name(name) - {} + template struct is_enum_list_attr : std::false_type {}; + template struct is_enum_list_attr> : std::true_type {}; - const std::string getFullName(const type_identity *item) const override { - return name + container_identity::getFullName(item); - } + template struct is_enum_field : std::false_type {}; + template struct is_enum_field> : std::true_type {}; - virtual bool is_readonly() const override { return true; } - virtual bool resize(void *ptr, int size) const override { return false; } - virtual bool erase(void *ptr, int size) const override { return false; } - virtual bool insert(void *ptr, int idx, void *item) const override { return false; } + // C strings are represented by char* and const char* + template + concept c_string = std::is_same_v || std::is_same_v; - protected: - virtual int item_count(void *ptr, CountMode) const override { return (int)((T*)ptr)->size(); } - virtual void *item_pointer(const type_identity *item, void *ptr, int idx) const override { - auto iter = (*(T*)ptr).begin(); - for (; idx > 0; idx--) ++iter; - return (void*)&*iter; - } - }; + // a std::vector of pointers; stored internally as std::vector + template + concept ptr_vector = is_std_vector::value && std::is_pointer_v; - template - class ro_stl_assoc_container_identity : public ro_stl_container_identity { - const type_identity *key_identity; - const type_identity *item_identity; + // std::vector and BitArray are bit containers + template + concept bit_container = is_bit_array::value || + (is_std_vector::value && std::is_same_v); - public: - ro_stl_assoc_container_identity(const char *name, const type_identity *key, const type_identity *item) - : ro_stl_container_identity(name, item), - key_identity(key), - item_identity(item) - {} + // sequence containers with mutable insert/erase/resize + template + concept seq_container = + (is_std_vector::value && !std::is_same_v && + !std::is_pointer_v) || + is_std_deque::value || is_df_array::value; - virtual const std::string getFullName(const type_identity*) const override { - return std::string(ro_stl_assoc_container_identity::name) + "<" + key_identity->getFullName() + ", " + item_identity->getFullName() + ">"; - } + // read-only containers without random access (associative types) + template + concept assoc_container = is_std_set::value || is_std_map::value; - protected: - virtual void *item_pointer(const type_identity *item, void *ptr, int idx) const override { - auto iter = (*(T*)ptr).begin(); - for (; idx > 0; idx--) ++iter; - return (void*)&iter->second; - } - }; + template + concept any_container = seq_container || assoc_container || + bit_container || ptr_vector || is_enum_list_attr::value; + + template + concept stl_string = std::is_same_v; + + template + concept fs_path = std::is_same_v; + + // df::bitfield_traits exists only for generated bitfield types + template + concept df_bitfield = requires { typename df::bitfield_traits::base_type; }; + + // generated compound types (struct/union/class) carry a static + // _identity member + template + concept has_identity_member = requires { T::_identity; }; + + // DfOtherVectors marks its descendants with a nested typedef + template + concept other_vectors = requires { typename T::dfhack_other_vectors; }; + + // an explicit df_identity_base typedef always wins; this lets the + // code generator (and hand-written types) pick the base directly + template + concept has_explicit_base = requires { typename T::df_identity_base; }; + + // The underlying storage actually manipulated by a container identity. + // std::vector uses std::vector, and BitArray uses + // BitArray, matching the assumptions of the original code. + template struct container_storage { using type = T; }; + template struct container_storage> { using type = std::vector; }; + template struct container_storage> { using type = BitArray; }; + template using container_storage_t = typename container_storage::type; + + template struct base_tag { using type = B; }; - class bit_array_identity : public bit_container_identity { - public: /* - * This class assumes that BitArray is equivalent - * in layout and behavior to BitArray for any T. + * The intermediate base for container-like T; implements the + * container_identity virtuals using constexpr dispatch on T. */ + template + class container_impl : public std::conditional_t< + bit_container, bit_container_identity, + std::conditional_t, ptr_container_identity, container_identity>> { + using cbase = std::conditional_t< + bit_container, bit_container_identity, + std::conditional_t, ptr_container_identity, container_identity>>; + using storage = container_storage_t; + + const char *name; + const type_identity *key_id; + + // enum_list_attr is a static attribute table and has no allocator + static constexpr TAllocateFn alloc_fn() { + if constexpr (is_enum_list_attr::value) + return NULL; + else + return &df::allocator_fn; + } - using container = BitArray; + public: + // sequence and read-only containers + container_impl(const char *name, const type_identity *item, const enum_identity *ienum = NULL) + requires (!bit_container && !is_std_map::value) + : cbase(sizeof(storage), alloc_fn(), item, ienum), name(name), key_id(NULL) + {} + + // bit containers take an index enum instead of an item identity + container_impl(const char *name, const enum_identity *ienum) + requires bit_container + : cbase(sizeof(storage), alloc_fn(), ienum), name(name), key_id(NULL) + {} + + // associative containers additionally take a key identity + container_impl(const char *name, const type_identity *key, const type_identity *item) + requires is_std_map::value + : cbase(sizeof(storage), alloc_fn(), item, NULL), name(name), key_id(key) + {} + + const std::string getFullName() const override { return getFullName(this->item); } + + const std::string getFullName(const type_identity *item) const override { + if constexpr (is_std_map::value) + return std::string(name) + "<" + key_id->getFullName() + ", " + item->getFullName() + ">"; + else if constexpr (is_bit_array::value) + return "BitArray<>"; + else + return std::string(name) + cbase::getFullName(item); + } - bit_array_identity(const enum_identity *ienum = NULL) - : bit_container_identity(sizeof(container), &allocator_fn, ienum) - {} + virtual bool is_readonly() const override { + return assoc_container; + } - virtual const std::string getFullName(const type_identity *item) const override { - return "BitArray<>"; - } + virtual bool resize(void *ptr, int size) const override { + if constexpr (is_bit_array::value) { + ((storage*)ptr)->resize((size+7)/8); + return true; + } + else if constexpr (std::is_same_v> || + seq_container || ptr_vector) { + (*(storage*)ptr).resize(size); + return true; + } + else + return false; + } + virtual bool erase(void *ptr, int index) const override { + if constexpr (seq_container || ptr_vector) { + auto &ct = *(storage*)ptr; + ct.erase(ct.begin()+index); + return true; + } + else + return false; + } + virtual bool insert(void *ptr, int index, void *pitem) const override { + if constexpr (ptr_vector) { + auto &ct = *(storage*)ptr; + ct.insert(ct.begin()+index, pitem); + return true; + } + else if constexpr (seq_container) { + auto &ct = *(storage*)ptr; + ct.insert(ct.begin()+index, *(typename T::value_type*)pitem); + return true; + } + else + return false; + } + virtual bool lua_insert2(lua_State* state, int fname_idx, void* ptr, int idx, int val_index) const override { + if constexpr (seq_container) { + using VT = typename T::value_type; + VT tmp{}; + auto id = (type_identity*)lua_touserdata(state, DFHack::LuaWrapper::UPVAL_ITEM_ID); + auto pitem = DFHack::LuaWrapper::get_object_internal(state, id, val_index, false); + bool useTemporary = (!pitem && id->isPrimitive()); + + if (useTemporary) + { + pitem = &tmp; + id->lua_write(state, fname_idx, pitem, val_index); + } + + if (id != this->item || !pitem) + DFHack::LuaWrapper::field_error(state, fname_idx, "incompatible object type", "insert"); + + return insert(ptr, idx, pitem); + } + else + return cbase::lua_insert2(state, fname_idx, ptr, idx, val_index); + } - virtual bool resize(void *ptr, int size) const override { - ((container*)ptr)->resize((size+7)/8); - return true; - } + protected: + virtual int item_count(void *ptr, container_identity::CountMode cnt) const override { + if constexpr (is_bit_array::value) + return cnt == container_identity::COUNT_LEN ? (int)((storage*)ptr)->size() * 8 : -1; + else if constexpr (is_enum_list_attr::value) + return cnt == container_identity::COUNT_WRITE ? 0 : (int)((storage*)ptr)->size; + else + return (int)((storage*)ptr)->size(); + } + virtual void *item_pointer(const type_identity *item, void *ptr, int idx) const override { + if constexpr (bit_container) + return NULL; + else if constexpr (is_enum_list_attr::value) + return (void*)&((storage*)ptr)->items[idx]; + else if constexpr (is_std_map::value) { + auto iter = (*(storage*)ptr).begin(); + for (; idx > 0; idx--) ++iter; + return (void*)&iter->second; + } + else if constexpr (assoc_container) { + auto iter = (*(storage*)ptr).begin(); + for (; idx > 0; idx--) ++iter; + return (void*)&*iter; + } + else + return &(*(storage*)ptr)[idx]; + } + // get_item/set_item only exist in bit_container_identity; for + // other bases these are ordinary (unused) member functions, and + // they implicitly override the virtuals for bit containers. + bool get_item(void *ptr, int idx) const { + if constexpr (is_bit_array::value) + return ((storage*)ptr)->is_set(idx); + else if constexpr (bit_container) + return (*(storage*)ptr)[idx]; + else + return false; + } + void set_item(void *ptr, int idx, bool val) const { + if constexpr (is_bit_array::value) + ((storage*)ptr)->set(idx, val); + else if constexpr (bit_container) + (*(storage*)ptr)[idx] = val; + } + }; - protected: - virtual int item_count(void *ptr, CountMode cnt) const override { - return cnt == COUNT_LEN ? ((container*)ptr)->size() * 8 : -1; - } - virtual bool get_item(void *ptr, int idx) const override { - return ((container*)ptr)->is_set(idx); - } - virtual void set_item(void *ptr, int idx, bool val) const override { - ((container*)ptr)->set(idx, val); + template + constexpr auto select_identity_base() { + if constexpr (has_explicit_base) + return base_tag{}; + else if constexpr (std::is_enum_v || is_enum_field::value) + return base_tag{}; + else if constexpr (df_bitfield) + return base_tag{}; + else if constexpr (has_identity_member) { + if constexpr (std::is_union_v) + return base_tag{}; + else if constexpr (other_vectors) + return base_tag{}; + else if constexpr (std::is_polymorphic_v) + return base_tag{}; + else + return base_tag{}; + } + else if constexpr (c_string) + return base_tag{}; + else if constexpr (std::is_same_v) + return base_tag{}; + else if constexpr (std::is_pointer_v) + return base_tag{}; + else if constexpr (std::is_arithmetic_v) + return base_tag{}; + else if constexpr (stl_string || fs_path) + return base_tag{}; + else if constexpr (std::is_array_v || is_std_array::value) + return base_tag{}; + else if constexpr (any_container) + return base_tag>{}; + else if constexpr (std::is_class_v) + return base_tag{}; + else + static_assert(!sizeof(T*), "type_identity_for: no identity category for this type"); } - }; -#endif - - class DFHACK_EXPORT stl_bit_vector_identity : public bit_container_identity { - public: - using container = std::vector; + } - stl_bit_vector_identity(const enum_identity *ienum = NULL) - : bit_container_identity(sizeof(container), &df::allocator_fn, ienum) - {} + template + using identity_base_of_t = typename decltype(detail::select_identity_base())::type; + + DFHACK_EXPORT void build_global_metatable(lua_State *state, const struct_identity *id); + DFHACK_EXPORT void lua_read_path(lua_State *state, void *ptr); + DFHACK_EXPORT void lua_write_path(lua_State *state, int fname_idx, void *ptr, int val_index); + + /* + * The identity of the C++ type T. The base class is selected by + * compile-time inspection of T (or an explicit df_identity_base + * typedef in T), and the remaining behavior is implemented here + * with constexpr dispatch on the properties of T. + */ + template + class type_identity_for : public identity_base_of_t { + using base = identity_base_of_t; - const std::string getFullName(const type_identity *item) const override { - return "vector" + bit_container_identity::getFullName(item); + public: + // forwards everything to the base class constructor; covers all + // compound identities, arrays, and opaque types + template + requires (sizeof...(Args) > 0 && std::is_constructible_v) + explicit type_identity_for(Args&&... args) : base(std::forward(args)...) {} + + // numbers take only a name; the size comes from T + explicit type_identity_for(const char *name) + requires std::is_same_v + : base(sizeof(T), name) {} + + // C strings + type_identity_for() + requires detail::c_string + : base(sizeof(T)) {} + + // std::string and std::filesystem::path + type_identity_for() + requires (detail::stl_string || detail::fs_path) + : base(sizeof(T), &df::allocator_fn) {} + + // pointers derive the target from identity_traits, or take an + // explicit target (used for void*) + type_identity_for() + requires (std::is_pointer_v && std::is_same_v) + : base(df::identity_traits>::get()) {} + + // named sequence/set containers + type_identity_for(const char *name, const type_identity *item, const enum_identity *ienum = NULL) + requires (std::is_same_v> && + !detail::bit_container && !detail::is_std_map::value) + : base(name, item, ienum) {} + + // named associative containers additionally take a key identity + type_identity_for(const char *name, const type_identity *key, const type_identity *item) + requires detail::is_std_map::value + : base(name, key, item) {} + + // std::vector + type_identity_for(const type_identity *item = NULL, const enum_identity *ienum = NULL) + requires detail::ptr_vector + : base("vector", item, ienum) {} + + // std::vector and BitArray + type_identity_for(const enum_identity *ienum = NULL) + requires detail::bit_container + : base("vector", ienum) {} + + // the global object wraps no actual type + explicit type_identity_for(const struct_field_info *fields) + requires std::is_same_v + : base(0, NULL, NULL, "global", NULL, fields) {} + + virtual identity_type type() const override { + if constexpr (detail::stl_string || detail::fs_path) + return IDTYPE_PRIMITIVE; + else if constexpr (detail::ptr_vector) + return IDTYPE_STL_PTR_VECTOR; + else if constexpr (std::is_same_v) + return IDTYPE_GLOBAL; + else + return base::type(); } - virtual bool resize(void *ptr, int size) const override { - (*(container*)ptr).resize(size); - return true; + virtual const std::string getFullName() const override { + if constexpr (detail::c_string) + return "char*"; + else if constexpr (detail::stl_string) + return "string"; + else if constexpr (detail::fs_path) + return "path"; + else + return base::getFullName(); } - protected: - virtual int item_count(void *ptr, CountMode) const override { - return (int)((container*)ptr)->size(); + virtual bool isPrimitive() const override { + if constexpr (detail::stl_string || detail::fs_path) + return true; + else + return base::isPrimitive(); } - virtual bool get_item(void *ptr, int idx) const override { - return (*(container*)ptr)[idx]; + + // isInteger only exists in number_identity_base; for other bases + // this is an ordinary member function, and it implicitly overrides + // the virtual for numbers. + bool isInteger() const { + return std::is_integral_v && !std::is_same_v; } - virtual void set_item(void *ptr, int idx, bool val) const override { - (*(container*)ptr)[idx] = val; + + virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override { + if constexpr (std::is_same_v) + lua_pushboolean(state, *(T*)ptr); + else if constexpr (std::is_floating_point_v) + lua_pushnumber(state, double(*(T*)ptr)); + else if constexpr (std::is_integral_v) + lua_pushinteger(state, int64_t(*(T*)ptr)); + else if constexpr (detail::c_string) { + auto pstr = *(T*)ptr; + if (pstr) + lua_pushstring(state, pstr); + else + lua_pushnil(state); + } + else if constexpr (detail::stl_string) { + auto pstr = (T*)ptr; + lua_pushlstring(state, pstr->data(), pstr->size()); + } + else if constexpr (detail::fs_path) + DFHack::lua_read_path(state, ptr); + else + base::lua_read(state, fname_idx, ptr); } - }; -#ifdef BUILD_DFHACK_LIB - template - class enum_list_attr_identity : public container_identity { - public: - using container = enum_list_attr; + virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override { + if constexpr (std::is_same_v) { + char *pb = (char*)ptr; - enum_list_attr_identity(const type_identity *item) - : container_identity(sizeof(container), NULL, item, NULL) - {} + if (lua_isboolean(state, val_index) || lua_isnil(state, val_index)) + *pb = lua_toboolean(state, val_index); + else if (lua_isnumber(state, val_index)) + *pb = lua_tointeger(state, val_index); + else + DFHack::LuaWrapper::field_error(state, fname_idx, "boolean or number expected", "write"); + } + else if constexpr (std::is_floating_point_v) { + if (!lua_isnumber(state, val_index)) + DFHack::LuaWrapper::field_error(state, fname_idx, "number expected", "write"); - const std::string getFullName(const type_identity *item) const override { - return "enum_list_attr" + container_identity::getFullName(item); + *(T*)ptr = T(lua_tonumber(state, val_index)); + } + else if constexpr (std::is_integral_v) { + int is_num = 0; + auto value = lua_tointegerx(state, val_index, &is_num); + if (!is_num) + DFHack::LuaWrapper::field_error(state, fname_idx, "integer expected", "write"); + *(T*)ptr = T(value); + } + else if constexpr (detail::c_string) + DFHack::LuaWrapper::field_error(state, fname_idx, "raw pointer string", "write"); + else if constexpr (detail::stl_string) { + size_t size; + const char *bytes = lua_tolstring(state, val_index, &size); + if (!bytes) + DFHack::LuaWrapper::field_error(state, fname_idx, "string expected", "write"); + + *(T*)ptr = std::string(bytes, size); + } + else if constexpr (detail::fs_path) + DFHack::lua_write_path(state, fname_idx, ptr, val_index); + else + base::lua_write(state, fname_idx, ptr, val_index); } - protected: - virtual int item_count(void *ptr, CountMode cm) const override { - return cm == COUNT_WRITE ? 0 : (int)((container*)ptr)->size; - } - virtual void *item_pointer(const type_identity *item, void *ptr, int idx) const override { - return (void*)&((container*)ptr)->items[idx]; + virtual void build_metatable(lua_State *state) const override { + if constexpr (std::is_same_v) + DFHack::build_global_metatable(state, this); + else + base::build_metatable(state); } }; -#endif +} + +namespace df +{ + using DFHack::type_identity_for; #define NUMBER_IDENTITY_TRAITS(type) \ template<> struct DFHACK_EXPORT identity_traits { \ static const bool is_primitive = true; \ - static const number_identity identity; \ - static const number_identity *get() { return &identity; } \ + static const type_identity_for identity; \ + static const type_identity_for *get() { return &identity; } \ }; // the space after the use of "type" in OPAQUE_IDENTITY_TRAITS is _required_ @@ -616,8 +753,8 @@ namespace df #define OPAQUE_IDENTITY_TRAITS(...) \ template<> struct DFHACK_EXPORT identity_traits<__VA_ARGS__ > { \ - static const opaque_identity identity; \ - static const opaque_identity *get() { return &identity; } \ + static const type_identity_for<__VA_ARGS__ > identity; \ + static const type_identity_for<__VA_ARGS__ > *get() { return &identity; } \ }; NUMBER_IDENTITY_TRAITS(char); @@ -663,41 +800,41 @@ namespace df template<> struct DFHACK_EXPORT identity_traits { static const bool is_primitive = true; - static const stl_string_identity identity; - static const stl_string_identity *get() { return &identity; } + static const type_identity_for identity; + static const type_identity_for *get() { return &identity; } }; template<> struct DFHACK_EXPORT identity_traits { static const bool is_primitive = true; - static const path_identity identity; - static const path_identity* get() { return &identity; } + static const type_identity_for identity; + static const type_identity_for* get() { return &identity; } }; template<> struct DFHACK_EXPORT identity_traits { static const bool is_primitive = true; - static const ptr_string_identity identity; - static const ptr_string_identity *get() { return &identity; } + static const type_identity_for identity; + static const type_identity_for *get() { return &identity; } }; template<> struct DFHACK_EXPORT identity_traits { static const bool is_primitive = true; - static const ptr_string_identity identity; - static const ptr_string_identity *get() { return &identity; } + static const type_identity_for identity; + static const type_identity_for *get() { return &identity; } }; template<> struct DFHACK_EXPORT identity_traits { static const bool is_primitive = true; - static const pointer_identity identity; - static const pointer_identity *get() { return &identity; } + static const type_identity_for identity; + static const type_identity_for *get() { return &identity; } }; template<> struct DFHACK_EXPORT identity_traits > { - static const stl_ptr_vector_identity identity; - static const stl_ptr_vector_identity *get() { return &identity; } + static const type_identity_for > identity; + static const type_identity_for > *get() { return &identity; } }; template<> struct DFHACK_EXPORT identity_traits > { - static const stl_bit_vector_identity identity; - static const stl_bit_vector_identity *get() { return &identity; } + static const type_identity_for > identity; + static const type_identity_for > *get() { return &identity; } }; #undef NUMBER_IDENTITY_TRAITS @@ -713,7 +850,7 @@ namespace df template struct identity_traits { static const bool is_primitive = true; - static const pointer_identity *get(); + static const pointer_identity_base *get(); }; #ifdef BUILD_DFHACK_LIB @@ -732,7 +869,7 @@ namespace df #endif template struct identity_traits > { - static const stl_ptr_vector_identity *get(); + static const ptr_container_identity *get(); }; // explicit specializations for these two types @@ -770,7 +907,7 @@ namespace df }; template<> struct identity_traits > { - static const bit_array_identity identity; + static const type_identity_for > identity; static const bit_container_identity *get() { return &identity; } }; @@ -792,54 +929,53 @@ namespace df #ifdef BUILD_DFHACK_LIB template inline const enum_identity *identity_traits >::get() { - static const enum_identity identity(identity_traits::get(), identity_traits::get()); + static const type_identity_for > identity(identity_traits::get(), identity_traits::get()); return &identity; } #endif template - inline const pointer_identity *identity_traits::get() { - static const pointer_identity identity; + inline const pointer_identity_base *identity_traits::get() { + static const type_identity_for identity; return &identity; } #ifdef BUILD_DFHACK_LIB template inline const container_identity *identity_traits::get() { - static const buffer_container_identity identity(sz, identity_traits::get()); + static const type_identity_for identity(sz, identity_traits::get()); return &identity; } template inline const container_identity* identity_traits>::get() { - static const buffer_container_identity identity(sz, identity_traits::get()); + static const type_identity_for > identity(int(sz), df::identity_traits::get()); return &identity; } template inline const container_identity *identity_traits >::get() { - using container = std::vector; - static const stl_container_identity identity("vector", identity_traits::get()); + static const type_identity_for > identity("vector", identity_traits::get()); return &identity; } #endif template - inline const stl_ptr_vector_identity *identity_traits >::get() { - static const stl_ptr_vector_identity identity(identity_traits::get()); + inline const ptr_container_identity *identity_traits >::get() { + static const type_identity_for > identity(identity_traits::get()); return &identity; } // explicit specializations for these two types // for availability in plugins - extern const DFHACK_EXPORT stl_container_identity > stl_vector_int32_t_identity; + extern const DFHACK_EXPORT type_identity_for > stl_vector_int32_t_identity; inline const container_identity* identity_traits >::get() { return &stl_vector_int32_t_identity; } - extern const DFHACK_EXPORT stl_container_identity > stl_vector_int16_t_identity; + extern const DFHACK_EXPORT type_identity_for > stl_vector_int16_t_identity; inline const container_identity* identity_traits >::get() { return &stl_vector_int16_t_identity; } @@ -847,56 +983,50 @@ namespace df #ifdef BUILD_DFHACK_LIB template inline const container_identity *identity_traits >::get() { - using container = std::deque; - static const stl_container_identity identity("deque", identity_traits::get()); + static const type_identity_for > identity("deque", identity_traits::get()); return &identity; } template inline const container_identity *identity_traits >::get() { - using container = std::set; - static const ro_stl_container_identity identity("set", identity_traits::get()); + static const type_identity_for > identity("set", identity_traits::get()); return &identity; } template inline const container_identity* identity_traits >::get() { - using container = std::unordered_set; - static const ro_stl_container_identity identity("unordered_set", identity_traits::get()); + static const type_identity_for > identity("unordered_set", identity_traits::get()); return &identity; } template inline const container_identity *identity_traits>::get() { - using container = std::map; - static const ro_stl_assoc_container_identity identity("map", identity_traits::get(), identity_traits::get()); + static const type_identity_for > identity("map", identity_traits::get(), identity_traits::get()); return &identity; } template inline const container_identity *identity_traits>::get() { - using container = std::unordered_map; - static const ro_stl_assoc_container_identity identity("unordered_map", identity_traits::get(), identity_traits::get()); + static const type_identity_for > identity("unordered_map", identity_traits::get(), identity_traits::get()); return &identity; } template inline const bit_container_identity *identity_traits >::get() { - static const bit_array_identity identity(identity_traits::get()); + static const type_identity_for > identity(identity_traits::get()); return &identity; } template inline const container_identity *identity_traits >::get() { - using container = DfArray; - static const stl_container_identity identity("DfArray", identity_traits::get()); + static const type_identity_for > identity("DfArray", identity_traits::get()); return &identity; } template inline const container_identity *identity_traits >::get() { - static const enum_list_attr_identity identity(identity_traits::get()); + static const type_identity_for > identity("enum_list_attr", identity_traits::get()); return &identity; } #endif diff --git a/library/include/LuaTools.h b/library/include/LuaTools.h index 4e40789c4a..f929c562f3 100644 --- a/library/include/LuaTools.h +++ b/library/include/LuaTools.h @@ -38,6 +38,7 @@ distribution. #include "Core.h" #include "ColorText.h" #include "DataDefs.h" +#include "DataIdentity.h" #include "df/coord.h" #include "df/coord2d.h" diff --git a/library/xml b/library/xml index d8ff8a6326..36fca2b189 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit d8ff8a6326db37c11dc4173fadcf30b56e51b42d +Subproject commit 36fca2b18989f5af30ea07de9c167428f2f7e6b8 diff --git a/plugins/devel/check-structures-sanity/types.cpp b/plugins/devel/check-structures-sanity/types.cpp index c0257640c7..84257a6ef7 100644 --- a/plugins/devel/check-structures-sanity/types.cpp +++ b/plugins/devel/check-structures-sanity/types.cpp @@ -156,13 +156,13 @@ bool CheckedStructure::has_type_at_offset(const CheckedStructure & type, size_t const type_identity *Checker::wrap_in_stl_ptr_vector(const type_identity *base) { - static std::map> wrappers; + static std::map>>> wrappers; auto it = wrappers.find(base); if (it != wrappers.end()) { return it->second.get(); } - return (wrappers[base] = std::make_unique(base, nullptr)).get(); + return (wrappers[base] = std::make_unique>>(base, nullptr)).get(); } const type_identity *Checker::wrap_in_pointer(const type_identity *base) diff --git a/plugins/stonesense b/plugins/stonesense index b71216ddfc..42e04ca70e 160000 --- a/plugins/stonesense +++ b/plugins/stonesense @@ -1 +1 @@ -Subproject commit b71216ddfc54e20ad48fb26f9ba1d82c7656b6ec +Subproject commit 42e04ca70e2e08c1fc1cef88837a7b7a38935490 diff --git a/scripts b/scripts index 4c4f5bf510..5d5fb16b81 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 4c4f5bf510e22fb96857328638e93ded3f608d0e +Subproject commit 5d5fb16b81b53cddfffe7b47a2f39f4f68272e52 From f8108894c9ade1f7a444e5938b4cf73818a3d9bc Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sun, 20 Sep 2026 03:22:21 -0500 Subject: [PATCH 15/42] use actual cast to make gcc happy --- library/include/DataIdentity.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/include/DataIdentity.h b/library/include/DataIdentity.h index 0bad6c4993..65d0a2183a 100644 --- a/library/include/DataIdentity.h +++ b/library/include/DataIdentity.h @@ -950,7 +950,7 @@ namespace df template inline const container_identity* identity_traits>::get() { - static const type_identity_for > identity(int(sz), df::identity_traits::get()); + static const type_identity_for > identity(static_cast(sz), df::identity_traits::get()); return &identity; } From 70c3ecb8c5d3dfcf3efa52bb25de0ef60ed1be2d Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sun, 20 Sep 2026 03:47:06 -0500 Subject: [PATCH 16/42] rewrite `container_impl` to be capability-based --- docs/dev/data-identity.rst | 8 +- library/include/DataIdentity.h | 196 +++++++++++++++++++++------------ 2 files changed, 132 insertions(+), 72 deletions(-) diff --git a/docs/dev/data-identity.rst b/docs/dev/data-identity.rst index 2300242ff9..0c638ffbaa 100644 --- a/docs/dev/data-identity.rst +++ b/docs/dev/data-identity.rst @@ -97,8 +97,12 @@ which derives from the appropriate runtime base for the C++ type ``T`` that it w * ``type_identity_for`` the leaf template that provides the identity of the C++ type ``T``. Its runtime base class is selected by compile-time inspection of ``T`` (or by a ``df_identity_base`` typedef in ``T``), covering all of the categories above: enums, bitfields, structs, unions, virtual classes, numbers, - pointers, C strings, ``std::string``, ``std::filesystem::path``, static arrays, STL containers, - ``BitArray``, ``DfArray``, ``enum_list_attr``, ``std::vector``, bit containers, and opaque types. + pointers, C strings, ``std::string``, ``std::filesystem::path``, static arrays, containers, and opaque + types. Container categories are detected by capability rather than by type name: any type that provides + index-addressable elements (``t[i]`` yielding a ``T&``), bit-element access (a bool proxy or + ``is_set``/``set`` accessors), mapped iteration (``mapped_type`` plus ``begin``/``end``), read-only + iteration, or a ``{ size, items[] }`` table (``enum_list_attr``) is treated as the corresponding kind of + container without being listed explicitly. ``global_identity`` is an alias for ``type_identity_for``, where ``global_object`` is an empty placeholder type: it holds, as a quasiobject, handles for all of the known Dwarf Fortress diff --git a/library/include/DataIdentity.h b/library/include/DataIdentity.h index 65d0a2183a..6f93114506 100644 --- a/library/include/DataIdentity.h +++ b/library/include/DataIdentity.h @@ -264,32 +264,12 @@ namespace DFHack * the appropriate behavior for the wrapped C++ type. */ - template struct is_std_vector : std::false_type {}; - template struct is_std_vector> : std::true_type {}; - - template struct is_std_deque : std::false_type {}; - template struct is_std_deque> : std::true_type {}; - template struct is_std_array : std::false_type {}; template struct is_std_array> : std::true_type {}; - template struct is_std_set : std::false_type {}; - template struct is_std_set> : std::true_type {}; - template struct is_std_set> : std::true_type {}; - - template struct is_std_map : std::false_type {}; - template struct is_std_map> : std::true_type {}; - template struct is_std_map> : std::true_type {}; - template struct is_bit_array : std::false_type {}; template struct is_bit_array> : std::true_type {}; - template struct is_df_array : std::false_type {}; - template struct is_df_array> : std::true_type {}; - - template struct is_enum_list_attr : std::false_type {}; - template struct is_enum_list_attr> : std::true_type {}; - template struct is_enum_field : std::false_type {}; template struct is_enum_field> : std::true_type {}; @@ -297,29 +277,95 @@ namespace DFHack template concept c_string = std::is_same_v || std::is_same_v; - // a std::vector of pointers; stored internally as std::vector + /* + * Container capability probes. These describe the behavioral axes + * that the container identity implementation dispatches on, so any + * type with a vector-like, array-like, bit-like, or associative + * interface is handled without being listed explicitly. + */ template - concept ptr_vector = is_std_vector::value && std::is_pointer_v; + concept has_value_type = requires { typename T::value_type; }; - // std::vector and BitArray are bit containers + // element count is available either as size() or as a size member template - concept bit_container = is_bit_array::value || - (is_std_vector::value && std::is_same_v); + concept sized = requires(const T& t) { { t.size() } -> std::convertible_to; } + || requires(const T& t) { { t.size } -> std::convertible_to; }; - // sequence containers with mutable insert/erase/resize + // dense random-access storage: t[i] yields a reference whose + // address is a value_type* (the vector/array contract) template - concept seq_container = - (is_std_vector::value && !std::is_same_v && - !std::is_pointer_v) || - is_std_deque::value || is_df_array::value; + concept random_access = has_value_type && sized && requires(T& t, int i) { + { &t[i] } -> std::convertible_to; + }; - // read-only containers without random access (associative types) + // indexable bit elements through a proxy object (vector and similar) template - concept assoc_container = is_std_set::value || is_std_map::value; + concept bit_indexable = has_value_type && sized && !random_access && + requires(T& t, int i, bool v) { + { t[i] } -> std::convertible_to; + t[i] = v; + }; + + // bit elements through explicit accessors; by convention such + // containers report byte counts from size() and resize() (BitArray). + // BitArray indexes by the enum type and is matched by name. + template + concept bit_accessors = is_bit_array::value || + (sized && requires(T& t, int i, bool v) { + { t.is_set(i) } -> std::convertible_to; + t.set(i, v); + }); + + // key-mapped iteration (std::map, std::unordered_map, and similar) + template + concept mapped = has_value_type && requires { typename T::mapped_type; } && + requires(T& t) { t.begin(); t.end(); }; + + // iterable but not index-addressable: read-only sequential access + // (std::set and similar) + template + concept ro_sequence = has_value_type && sized && + requires(T& t) { t.begin(); t.end(); } && + !random_access && !bit_indexable && !mapped; + + // an aggregate exposing a fixed { size, items[] } table (enum_list_attr) + template + concept item_list = sized && requires(const T& t, int i) { t.items[i]; }; + // bit containers of either flavor; mapped types that happen to + // satisfy the proxy probes (e.g. map) are excluded template - concept any_container = seq_container || assoc_container || - bit_container || ptr_vector || is_enum_list_attr::value; + concept bit_container = (bit_indexable || bit_accessors) && !mapped; + + // sequence containers with mutable, index-addressable elements + template + concept seq_container = random_access && + !std::is_pointer_v; + + // containers of pointer elements use pointer-container semantics + // and are stored internally as containers of void* + template + concept ptr_container = random_access && + std::is_pointer_v; + + // read-only containers accessed by iteration + template + concept assoc_container = ro_sequence || mapped; + + template + concept any_container = seq_container || ptr_container || + bit_container || assoc_container || item_list; + + // mutable-sequence operations + template + concept resizable = requires(T& t, int n) { t.resize(n); }; + + template + concept index_erasable = requires(T& t, int i) { t.erase(t.begin() + i); }; + + template + concept index_insertable = has_value_type && + requires(T& t, int i, typename T::value_type v) { t.insert(t.begin() + i, v); }; template concept stl_string = std::is_same_v; @@ -346,11 +392,14 @@ namespace DFHack concept has_explicit_base = requires { typename T::df_identity_base; }; // The underlying storage actually manipulated by a container identity. - // std::vector uses std::vector, and BitArray uses - // BitArray, matching the assumptions of the original code. + // Pointer-element containers are manipulated as containers of void*, + // and BitArray is assumed layout-equivalent to BitArray, + // matching the assumptions of the original code. template struct container_storage { using type = T; }; - template struct container_storage> { using type = std::vector; }; template struct container_storage> { using type = BitArray; }; + template class C, typename E, typename... A> + requires (sizeof(C) > 0) + struct container_storage> { using type = C; }; template using container_storage_t = typename container_storage::type; template struct base_tag { using type = B; }; @@ -362,27 +411,35 @@ namespace DFHack template class container_impl : public std::conditional_t< bit_container, bit_container_identity, - std::conditional_t, ptr_container_identity, container_identity>> { + std::conditional_t, ptr_container_identity, container_identity>> { using cbase = std::conditional_t< bit_container, bit_container_identity, - std::conditional_t, ptr_container_identity, container_identity>>; + std::conditional_t, ptr_container_identity, container_identity>>; using storage = container_storage_t; const char *name; const type_identity *key_id; - // enum_list_attr is a static attribute table and has no allocator + // item tables like enum_list_attr describe static external + // data and have no allocator static constexpr TAllocateFn alloc_fn() { - if constexpr (is_enum_list_attr::value) + if constexpr (item_list) return NULL; else return &df::allocator_fn; } + static size_t element_count(const storage &ct) { + if constexpr (requires { { ct.size() } -> std::convertible_to; }) + return (size_t)ct.size(); + else + return (size_t)ct.size; + } + public: // sequence and read-only containers container_impl(const char *name, const type_identity *item, const enum_identity *ienum = NULL) - requires (!bit_container && !is_std_map::value) + requires (!bit_container && !mapped) : cbase(sizeof(storage), alloc_fn(), item, ienum), name(name), key_id(NULL) {} @@ -392,16 +449,16 @@ namespace DFHack : cbase(sizeof(storage), alloc_fn(), ienum), name(name), key_id(NULL) {} - // associative containers additionally take a key identity + // mapped containers additionally take a key identity container_impl(const char *name, const type_identity *key, const type_identity *item) - requires is_std_map::value + requires mapped : cbase(sizeof(storage), alloc_fn(), item, NULL), name(name), key_id(key) {} const std::string getFullName() const override { return getFullName(this->item); } const std::string getFullName(const type_identity *item) const override { - if constexpr (is_std_map::value) + if constexpr (mapped) return std::string(name) + "<" + key_id->getFullName() + ", " + item->getFullName() + ">"; else if constexpr (is_bit_array::value) return "BitArray<>"; @@ -414,12 +471,11 @@ namespace DFHack } virtual bool resize(void *ptr, int size) const override { - if constexpr (is_bit_array::value) { + if constexpr (bit_accessors) { ((storage*)ptr)->resize((size+7)/8); return true; } - else if constexpr (std::is_same_v> || - seq_container || ptr_vector) { + else if constexpr (resizable) { (*(storage*)ptr).resize(size); return true; } @@ -427,7 +483,7 @@ namespace DFHack return false; } virtual bool erase(void *ptr, int index) const override { - if constexpr (seq_container || ptr_vector) { + if constexpr (index_erasable) { auto &ct = *(storage*)ptr; ct.erase(ct.begin()+index); return true; @@ -436,12 +492,12 @@ namespace DFHack return false; } virtual bool insert(void *ptr, int index, void *pitem) const override { - if constexpr (ptr_vector) { + if constexpr (ptr_container && index_insertable) { auto &ct = *(storage*)ptr; ct.insert(ct.begin()+index, pitem); return true; } - else if constexpr (seq_container) { + else if constexpr (!bit_container && index_insertable) { auto &ct = *(storage*)ptr; ct.insert(ct.begin()+index, *(typename T::value_type*)pitem); return true; @@ -450,7 +506,7 @@ namespace DFHack return false; } virtual bool lua_insert2(lua_State* state, int fname_idx, void* ptr, int idx, int val_index) const override { - if constexpr (seq_container) { + if constexpr (seq_container && index_insertable) { using VT = typename T::value_type; VT tmp{}; auto id = (type_identity*)lua_touserdata(state, DFHack::LuaWrapper::UPVAL_ITEM_ID); @@ -474,24 +530,24 @@ namespace DFHack protected: virtual int item_count(void *ptr, container_identity::CountMode cnt) const override { - if constexpr (is_bit_array::value) - return cnt == container_identity::COUNT_LEN ? (int)((storage*)ptr)->size() * 8 : -1; - else if constexpr (is_enum_list_attr::value) - return cnt == container_identity::COUNT_WRITE ? 0 : (int)((storage*)ptr)->size; + if constexpr (bit_accessors) + return cnt == container_identity::COUNT_LEN ? (int)(element_count(*(storage*)ptr) * 8) : -1; + else if constexpr (item_list) + return cnt == container_identity::COUNT_WRITE ? 0 : (int)element_count(*(storage*)ptr); else - return (int)((storage*)ptr)->size(); + return (int)element_count(*(storage*)ptr); } virtual void *item_pointer(const type_identity *item, void *ptr, int idx) const override { if constexpr (bit_container) return NULL; - else if constexpr (is_enum_list_attr::value) + else if constexpr (item_list) return (void*)&((storage*)ptr)->items[idx]; - else if constexpr (is_std_map::value) { + else if constexpr (mapped) { auto iter = (*(storage*)ptr).begin(); for (; idx > 0; idx--) ++iter; return (void*)&iter->second; } - else if constexpr (assoc_container) { + else if constexpr (ro_sequence) { auto iter = (*(storage*)ptr).begin(); for (; idx > 0; idx--) ++iter; return (void*)&*iter; @@ -503,17 +559,17 @@ namespace DFHack // other bases these are ordinary (unused) member functions, and // they implicitly override the virtuals for bit containers. bool get_item(void *ptr, int idx) const { - if constexpr (is_bit_array::value) + if constexpr (bit_accessors) return ((storage*)ptr)->is_set(idx); - else if constexpr (bit_container) + else if constexpr (bit_indexable) return (*(storage*)ptr)[idx]; else return false; } void set_item(void *ptr, int idx, bool val) const { - if constexpr (is_bit_array::value) + if constexpr (bit_accessors) ((storage*)ptr)->set(idx, val); - else if constexpr (bit_container) + else if constexpr (bit_indexable) (*(storage*)ptr)[idx] = val; } }; @@ -605,17 +661,17 @@ namespace DFHack // named sequence/set containers type_identity_for(const char *name, const type_identity *item, const enum_identity *ienum = NULL) requires (std::is_same_v> && - !detail::bit_container && !detail::is_std_map::value) + !detail::bit_container && !detail::mapped) : base(name, item, ienum) {} - // named associative containers additionally take a key identity + // named mapped containers additionally take a key identity type_identity_for(const char *name, const type_identity *key, const type_identity *item) - requires detail::is_std_map::value + requires detail::mapped : base(name, key, item) {} - // std::vector + // containers of pointer elements type_identity_for(const type_identity *item = NULL, const enum_identity *ienum = NULL) - requires detail::ptr_vector + requires detail::ptr_container : base("vector", item, ienum) {} // std::vector and BitArray @@ -631,7 +687,7 @@ namespace DFHack virtual identity_type type() const override { if constexpr (detail::stl_string || detail::fs_path) return IDTYPE_PRIMITIVE; - else if constexpr (detail::ptr_vector) + else if constexpr (detail::ptr_container) return IDTYPE_STL_PTR_VECTOR; else if constexpr (std::is_same_v) return IDTYPE_GLOBAL; From 688c709b006d643bc9d0da5bf89348a6b3147c7b Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sun, 20 Sep 2026 04:01:53 -0500 Subject: [PATCH 17/42] export some symbols that gcc needs this is due to more aggressive devirtualization by gcc --- library/include/DataIdentity.h | 2 +- library/include/LuaWrapper.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/library/include/DataIdentity.h b/library/include/DataIdentity.h index 6f93114506..424d8e131d 100644 --- a/library/include/DataIdentity.h +++ b/library/include/DataIdentity.h @@ -225,7 +225,7 @@ namespace df * Unlike other container identities the item count is fixed, * so instances can also serve as the identity of ad-hoc buffers. */ - class buffer_container_identity : public container_identity { + class DFHACK_EXPORT buffer_container_identity : public container_identity { int size; public: diff --git a/library/include/LuaWrapper.h b/library/include/LuaWrapper.h index c913d05893..9c9c3d6ee7 100644 --- a/library/include/LuaWrapper.h +++ b/library/include/LuaWrapper.h @@ -139,13 +139,13 @@ namespace LuaWrapper { /** * Report an error while accessing a field (index = field name). */ - [[noreturn]] void field_error(lua_State *state, int index, const char *err, const char *mode); + [[noreturn]] DFHACK_EXPORT void field_error(lua_State *state, int index, const char *err, const char *mode); /* * If is_method is true, these use UPVAL_TYPETABLE to save a hash lookup. */ void push_object_internal(lua_State *state, const type_identity *type, void *ptr, bool in_method = true); - void *get_object_internal(lua_State *state, const type_identity *type, int val_index, bool exact_type, bool in_method = true); + DFHACK_EXPORT void *get_object_internal(lua_State *state, const type_identity *type, int val_index, bool exact_type, bool in_method = true); void push_adhoc_pointer(lua_State *state, void *ptr, const type_identity *target); From 6344fbd21116abf677b8e73093627c61d9da7926 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Tue, 13 Oct 2026 16:03:26 -0500 Subject: [PATCH 18/42] simplify global identity handling --- docs/dev/data-identity.rst | 2 ++ library/LuaTypes.cpp | 4 ++-- library/include/DataDefs.h | 11 ++++++++++- library/include/DataIdentity.h | 10 ---------- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/dev/data-identity.rst b/docs/dev/data-identity.rst index 0c638ffbaa..bfca2388e4 100644 --- a/docs/dev/data-identity.rst +++ b/docs/dev/data-identity.rst @@ -70,6 +70,8 @@ which derives from the appropriate runtime base for the C++ type ``T`` that it w * ``virtual_identity`` polymorphic C++ ``class`` or ``structure`` having a virtual table to handle virtual dispatch + * ``global_identity_base`` special-case identity for the global quasiobject + * ``xlsx_file_handle_identity`` special case * ``xlsx_sheet_handle_identity`` special case diff --git a/library/LuaTypes.cpp b/library/LuaTypes.cpp index c9922689ca..45d1f8e8e3 100644 --- a/library/LuaTypes.cpp +++ b/library/LuaTypes.cpp @@ -1697,10 +1697,10 @@ void other_vectors_identity::build_metatable(lua_State *state) const SetPtrMethods(state, base+1, base+2); } -void DFHack::build_global_metatable(lua_State *state, const struct_identity *id) +void global_identity_base::build_metatable(lua_State *state) const { int base = lua_gettop(state); - MakeFieldMetatable(state, id, meta_global_index, meta_global_newindex, meta_struct_next, true); + MakeFieldMetatable(state, this, meta_global_index, meta_global_newindex, meta_struct_next, true); SetStructMethod(state, base+1, base+2, meta_global_field_reference, "_field"); SetPtrMethods(state, base+1, base+2); } diff --git a/library/include/DataDefs.h b/library/include/DataDefs.h index 8ff99c750c..fd7fdc6521 100644 --- a/library/include/DataDefs.h +++ b/library/include/DataDefs.h @@ -329,10 +329,19 @@ namespace DFHack bool is_equivalent(const struct_identity* other) const; }; + class DFHACK_EXPORT global_identity_base : public struct_identity { + public: + using struct_identity::struct_identity; + + virtual identity_type type() const override { return IDTYPE_GLOBAL; } + + virtual void build_metatable(lua_State *state) const override; + }; + // Placeholder type wrapped by the identity of the global "object", // which does not correspond to an actual C++ type. struct global_object { - using df_identity_base = struct_identity; + using df_identity_base = global_identity_base; }; using global_identity = type_identity_for; diff --git a/library/include/DataIdentity.h b/library/include/DataIdentity.h index 424d8e131d..c2b9667cc5 100644 --- a/library/include/DataIdentity.h +++ b/library/include/DataIdentity.h @@ -616,7 +616,6 @@ namespace DFHack template using identity_base_of_t = typename decltype(detail::select_identity_base())::type; - DFHACK_EXPORT void build_global_metatable(lua_State *state, const struct_identity *id); DFHACK_EXPORT void lua_read_path(lua_State *state, void *ptr); DFHACK_EXPORT void lua_write_path(lua_State *state, int fname_idx, void *ptr, int val_index); @@ -689,8 +688,6 @@ namespace DFHack return IDTYPE_PRIMITIVE; else if constexpr (detail::ptr_container) return IDTYPE_STL_PTR_VECTOR; - else if constexpr (std::is_same_v) - return IDTYPE_GLOBAL; else return base::type(); } @@ -783,13 +780,6 @@ namespace DFHack else base::lua_write(state, fname_idx, ptr, val_index); } - - virtual void build_metatable(lua_State *state) const override { - if constexpr (std::is_same_v) - DFHack::build_global_metatable(state, this); - else - base::build_metatable(state); - } }; } From 93ef1abc563377ec09895e4d97025a9ca44da937 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Tue, 13 Oct 2026 17:26:29 -0500 Subject: [PATCH 19/42] other restructuring restructure other `type_identity_for` methods to reduce the `if constexpr` chains that were there before also, revise the function call wrappers to reduce virtual dispatches during argument marshalling --- library/include/DataFuncs.h | 45 +++++++- library/include/DataIdentity.h | 193 ++++++++++++++++++++++----------- 2 files changed, 167 insertions(+), 71 deletions(-) diff --git a/library/include/DataFuncs.h b/library/include/DataFuncs.h index 35db38dc4d..27ad5f2e7f 100644 --- a/library/include/DataFuncs.h +++ b/library/include/DataFuncs.h @@ -44,6 +44,43 @@ namespace df { template concept isPrimitive = identity_traits::is_primitive || std::is_enum_v || std::is_void_v; + namespace detail { + /* + * Marshalling for wrapped-function arguments and return values that + * avoids virtual dispatch through the type identity. Types with a + * value representation (numbers, strings, enums, etc.) are handled + * directly by the lua_*_value overloads; pointers use the static + * pointer_identity_base implementation with the target identity + * supplied explicitly; anything else falls back to the identity's + * virtual method. + */ + template + inline void marshal_from_lua(lua_State *L, int idx, T &out) + { + using DFHack::LuaWrapper::UPVAL_METHOD_NAME; + if constexpr (requires { DFHack::detail::lua_write_value(L, UPVAL_METHOD_NAME, out, idx); }) + DFHack::detail::lua_write_value(L, UPVAL_METHOD_NAME, out, idx); + else if constexpr (std::is_same_v, DFHack::pointer_identity_base>) + { + const type_identity *target = nullptr; + if constexpr (requires { df::identity_traits>::get(); }) + target = df::identity_traits>::get(); + DFHack::pointer_identity_base::lua_write(L, UPVAL_METHOD_NAME, &out, target, idx); + } + else + df::identity_traits::get()->lua_write(L, UPVAL_METHOD_NAME, &out, idx); + } + + template + inline void marshal_to_lua(lua_State *L, T &v) + { + if constexpr (requires { DFHack::detail::lua_read_value(L, v); }) + DFHack::detail::lua_read_value(L, v); + else + df::identity_traits::get()->lua_read(L, DFHack::LuaWrapper::UPVAL_METHOD_NAME, &v); + } + } + // handle call-by-reference function arguments by converting pointers to a reference // will throw if lua code tries to pass a null pointer template requires std::is_reference_v @@ -53,7 +90,7 @@ namespace df { using DFHack::LuaWrapper::UPVAL_METHOD_NAME; using Ptr = std::add_pointer_t>; Ptr ptr{}; - df::identity_traits::get()->lua_write(L, UPVAL_METHOD_NAME, &ptr, idx); + detail::marshal_from_lua(L, idx, ptr); if (ptr == nullptr) { field_error(L, UPVAL_METHOD_NAME, "cannot convert null pointer to reference", "call"); @@ -66,9 +103,8 @@ namespace df { template T get_from_lua_state(lua_State* L, int idx) { - using DFHack::LuaWrapper::UPVAL_METHOD_NAME; T val{}; - df::identity_traits::get()->lua_write(L, UPVAL_METHOD_NAME, &val, idx); + detail::marshal_from_lua(L, idx, val); return val; } @@ -76,7 +112,6 @@ namespace df { requires std::is_invocable_r_v void call_and_push_impl(lua_State* L, int base, std::index_sequence, FT fun, ET... extra) { - using DFHack::LuaWrapper::UPVAL_METHOD_NAME; if constexpr (std::is_same_v) { std::invoke(fun, extra..., (get_from_lua_state(L, base+I))...); lua_pushnil(L); @@ -84,7 +119,7 @@ namespace df { else { RT rv = std::invoke(fun, extra..., (get_from_lua_state(L, base+I))...); - df::identity_traits::get()->lua_read(L, UPVAL_METHOD_NAME, &rv); + detail::marshal_to_lua(L, rv); } } diff --git a/library/include/DataIdentity.h b/library/include/DataIdentity.h index c2b9667cc5..9e014b5556 100644 --- a/library/include/DataIdentity.h +++ b/library/include/DataIdentity.h @@ -25,6 +25,7 @@ distribution. #pragma once #include +#include #include #include #include @@ -619,6 +620,122 @@ namespace DFHack DFHACK_EXPORT void lua_read_path(lua_State *state, void *ptr); DFHACK_EXPORT void lua_write_path(lua_State *state, int fname_idx, void *ptr, int val_index); + namespace detail { + /* + * Per-category overrides of type_identity properties, selected at + * compile time by constraint resolution. Categories with no + * matching overload fall back to the identity base class's + * implementation in type_identity_for's virtuals. + */ + + // type() overrides + template requires (stl_string || fs_path) + constexpr identity_type type_override() { return IDTYPE_PRIMITIVE; } + template requires ptr_container + constexpr identity_type type_override() { return IDTYPE_STL_PTR_VECTOR; } + + // getFullName() overrides + template requires c_string + constexpr const char *name_override() { return "char*"; } + template requires stl_string + constexpr const char *name_override() { return "string"; } + template requires fs_path + constexpr const char *name_override() { return "path"; } + + // isPrimitive() override + template requires (stl_string || fs_path) + constexpr bool primitive_override() { return true; } + + /* + * Marshalling of primitive values between C++ objects and Lua. + * Overload resolution on the exact deduced type of the C++ object + * selects the implementation, so support for a new value type is + * added by writing a new overload here. Types with no matching + * overload fall back to the identity base class's implementation + * (reference semantics). All parameters are deduced, so implicit + * conversions of the wrapped type are never considered. + */ + template requires std::same_as + inline void lua_read_value(lua_State *state, const T &v) { + lua_pushboolean(state, v); + } + template requires (std::integral && !std::same_as) + inline void lua_read_value(lua_State *state, const T &v) { + lua_pushinteger(state, int64_t(v)); + } + template requires std::floating_point + inline void lua_read_value(lua_State *state, const T &v) { + lua_pushnumber(state, double(v)); + } + // enums marshal through their underlying type, matching + // enum_identity's forwarding to base_type + template requires std::is_enum_v + inline void lua_read_value(lua_State *state, const T &v) { + lua_read_value(state, static_cast>(v)); + } + template requires c_string + inline void lua_read_value(lua_State *state, const T &v) { + if (v) + lua_pushstring(state, v); + else + lua_pushnil(state); + } + template requires stl_string + inline void lua_read_value(lua_State *state, const T &v) { + lua_pushlstring(state, v.data(), v.size()); + } + template requires fs_path + inline void lua_read_value(lua_State *state, const T &v) { + lua_read_path(state, const_cast(&v)); + } + + template requires std::same_as + inline void lua_write_value(lua_State *state, int fname_idx, T &out, int val_index) { + if (lua_isboolean(state, val_index) || lua_isnil(state, val_index)) + out = lua_toboolean(state, val_index); + else if (lua_isnumber(state, val_index)) + out = lua_tointeger(state, val_index) != 0; + else + LuaWrapper::field_error(state, fname_idx, "boolean or number expected", "write"); + } + template requires (std::integral && !std::same_as) + inline void lua_write_value(lua_State *state, int fname_idx, T &out, int val_index) { + int is_num = 0; + auto value = lua_tointegerx(state, val_index, &is_num); + if (!is_num) + LuaWrapper::field_error(state, fname_idx, "integer expected", "write"); + out = T(value); + } + template requires std::floating_point + inline void lua_write_value(lua_State *state, int fname_idx, T &out, int val_index) { + if (!lua_isnumber(state, val_index)) + LuaWrapper::field_error(state, fname_idx, "number expected", "write"); + out = T(lua_tonumber(state, val_index)); + } + template requires std::is_enum_v + inline void lua_write_value(lua_State *state, int fname_idx, T &out, int val_index) { + std::underlying_type_t tmp; + lua_write_value(state, fname_idx, tmp, val_index); + out = T(tmp); + } + template requires c_string + inline void lua_write_value(lua_State *state, int fname_idx, T &, int) { + LuaWrapper::field_error(state, fname_idx, "raw pointer string", "write"); + } + template requires stl_string + inline void lua_write_value(lua_State *state, int fname_idx, T &out, int val_index) { + size_t size; + const char *bytes = lua_tolstring(state, val_index, &size); + if (!bytes) + LuaWrapper::field_error(state, fname_idx, "string expected", "write"); + out.assign(bytes, size); + } + template requires fs_path + inline void lua_write_value(lua_State *state, int fname_idx, T &out, int val_index) { + lua_write_path(state, fname_idx, &out, val_index); + } + } + /* * The identity of the C++ type T. The base class is selected by * compile-time inspection of T (or an explicit df_identity_base @@ -684,28 +801,22 @@ namespace DFHack : base(0, NULL, NULL, "global", NULL, fields) {} virtual identity_type type() const override { - if constexpr (detail::stl_string || detail::fs_path) - return IDTYPE_PRIMITIVE; - else if constexpr (detail::ptr_container) - return IDTYPE_STL_PTR_VECTOR; + if constexpr (requires { detail::type_override(); }) + return detail::type_override(); else return base::type(); } virtual const std::string getFullName() const override { - if constexpr (detail::c_string) - return "char*"; - else if constexpr (detail::stl_string) - return "string"; - else if constexpr (detail::fs_path) - return "path"; + if constexpr (requires { detail::name_override(); }) + return detail::name_override(); else return base::getFullName(); } virtual bool isPrimitive() const override { - if constexpr (detail::stl_string || detail::fs_path) - return true; + if constexpr (requires { detail::primitive_override(); }) + return detail::primitive_override(); else return base::isPrimitive(); } @@ -718,65 +829,15 @@ namespace DFHack } virtual void lua_read(lua_State *state, int fname_idx, void *ptr) const override { - if constexpr (std::is_same_v) - lua_pushboolean(state, *(T*)ptr); - else if constexpr (std::is_floating_point_v) - lua_pushnumber(state, double(*(T*)ptr)); - else if constexpr (std::is_integral_v) - lua_pushinteger(state, int64_t(*(T*)ptr)); - else if constexpr (detail::c_string) { - auto pstr = *(T*)ptr; - if (pstr) - lua_pushstring(state, pstr); - else - lua_pushnil(state); - } - else if constexpr (detail::stl_string) { - auto pstr = (T*)ptr; - lua_pushlstring(state, pstr->data(), pstr->size()); - } - else if constexpr (detail::fs_path) - DFHack::lua_read_path(state, ptr); + if constexpr (requires { detail::lua_read_value(state, *(const T*)ptr); }) + detail::lua_read_value(state, *(const T*)ptr); else base::lua_read(state, fname_idx, ptr); } virtual void lua_write(lua_State *state, int fname_idx, void *ptr, int val_index) const override { - if constexpr (std::is_same_v) { - char *pb = (char*)ptr; - - if (lua_isboolean(state, val_index) || lua_isnil(state, val_index)) - *pb = lua_toboolean(state, val_index); - else if (lua_isnumber(state, val_index)) - *pb = lua_tointeger(state, val_index); - else - DFHack::LuaWrapper::field_error(state, fname_idx, "boolean or number expected", "write"); - } - else if constexpr (std::is_floating_point_v) { - if (!lua_isnumber(state, val_index)) - DFHack::LuaWrapper::field_error(state, fname_idx, "number expected", "write"); - - *(T*)ptr = T(lua_tonumber(state, val_index)); - } - else if constexpr (std::is_integral_v) { - int is_num = 0; - auto value = lua_tointegerx(state, val_index, &is_num); - if (!is_num) - DFHack::LuaWrapper::field_error(state, fname_idx, "integer expected", "write"); - *(T*)ptr = T(value); - } - else if constexpr (detail::c_string) - DFHack::LuaWrapper::field_error(state, fname_idx, "raw pointer string", "write"); - else if constexpr (detail::stl_string) { - size_t size; - const char *bytes = lua_tolstring(state, val_index, &size); - if (!bytes) - DFHack::LuaWrapper::field_error(state, fname_idx, "string expected", "write"); - - *(T*)ptr = std::string(bytes, size); - } - else if constexpr (detail::fs_path) - DFHack::lua_write_path(state, fname_idx, ptr, val_index); + if constexpr (requires { detail::lua_write_value(state, fname_idx, *(T*)ptr, val_index); }) + detail::lua_write_value(state, fname_idx, *(T*)ptr, val_index); else base::lua_write(state, fname_idx, ptr, val_index); } From b1e7adaa4b67efb9c359c758225d2af984d90dc8 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Wed, 28 Oct 2026 16:20:45 -0500 Subject: [PATCH 20/42] fix submodules --- plugins/stonesense | 2 +- scripts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/stonesense b/plugins/stonesense index 42e04ca70e..b71216ddfc 160000 --- a/plugins/stonesense +++ b/plugins/stonesense @@ -1 +1 @@ -Subproject commit 42e04ca70e2e08c1fc1cef88837a7b7a38935490 +Subproject commit b71216ddfc54e20ad48fb26f9ba1d82c7656b6ec diff --git a/scripts b/scripts index 5d5fb16b81..4c4f5bf510 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 5d5fb16b81b53cddfffe7b47a2f39f4f68272e52 +Subproject commit 4c4f5bf510e22fb96857328638e93ded3f608d0e From 5bdb27b24a3dcf8ce3f7062d4d24eddc37488b40 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Wed, 28 Oct 2026 16:22:29 -0500 Subject: [PATCH 21/42] update xml --- library/xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/xml b/library/xml index 36fca2b189..1196d462d1 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 36fca2b18989f5af30ea07de9c167428f2f7e6b8 +Subproject commit 1196d462d1c16ba0208be9eb7c58b07e2e955452 From 75031341942d174132ef923f653f88f33bd96967 Mon Sep 17 00:00:00 2001 From: SilasD Date: Wed, 28 Oct 2026 15:51:35 -0700 Subject: [PATCH 22/42] Units.cpp teleport -- break links to cages and chains. --- library/modules/Units.cpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/library/modules/Units.cpp b/library/modules/Units.cpp index 3d1a903797..9a88a2d000 100644 --- a/library/modules/Units.cpp +++ b/library/modules/Units.cpp @@ -39,6 +39,8 @@ distribution. #include "modules/World.h" #include "df/activity_entry.h" +#include "df/building_cagest.h" +#include "df/building_chainst.h" #include "df/burrow.h" #include "df/caste_raw.h" #include "df/creature_interaction_effect_display_namest.h" @@ -872,6 +874,38 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) else rider->pos = unit->pos; + // If the unit was teleported out of a cage, break those links. + if (auto cage = Units::getContainer(unit)) + { + for (auto job: strict_virtual_cast(Items::getHolderBuilding(cage))->jobs) + if (job->job_type == df::job_type::ReleaseLargeCreature || job->job_type == df::job_type::ReleasePet) + if (auto gref = findRef(job->general_refs, df::general_ref_type::UNIT_CAGEE); + gref && gref->getID() == unit->id) + Job::removeJob(job); + if (auto bld = strict_virtual_cast(Items::getHolderBuilding(cage))) + std::erase(bld->assigned_units, unit->id); + removeRef(cage->general_refs, df::general_ref_type::CONTAINS_UNIT, unit->id); + removeRef(unit->general_refs, df::general_ref_type::CONTAINED_IN_ITEM, cage->id); + unit->flags1.bits.caged = false; + } + + // If the unit was teleported out of a chain, break those links. + if (auto bld = strict_virtual_cast + (findBuildingRef(unit->general_refs, df::general_ref_type::BUILDING_CHAIN))) + { + for (auto job: bld->jobs) + if (auto gref = findRef(job->general_refs, df::general_ref_type::UNIT_CAGEE); + (job->job_type == df::job_type::UnchainAnimal || job->job_type == df::job_type::UnchainPet) + && gref && gref->getID() == unit->id) + Job::removeJob(job); + if (bld->chained == unit) + bld->chained = NULL; + if (bld->assigned == unit) + bld->assigned = NULL; + removeRef(unit->general_refs, df::general_ref_type::BUILDING_CHAIN, bld->id); + unit->flags1.bits.chained = false; + } + return true; } From 0fd30f69375300efee5180a388808aa72e643dd0 Mon Sep 17 00:00:00 2001 From: SilasD Date: Thu, 29 Oct 2026 17:55:51 -0700 Subject: [PATCH 23/42] Units.cpp if teleporting a rider, unlink from mount. --- library/modules/Units.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/library/modules/Units.cpp b/library/modules/Units.cpp index 9a88a2d000..8addee7e43 100644 --- a/library/modules/Units.cpp +++ b/library/modules/Units.cpp @@ -874,6 +874,26 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) else rider->pos = unit->pos; + // if the unit itself was a rider, break that link. + if (unit->flags1.bits.rider) + { + auto mount = df::unit::find(unit->relationship_ids[df::unit_relationship_type::RiderMount]); + unit->flags1.bits.rider = false; + unit->relationship_ids[df::unit_relationship_type::RiderMount] = -1; + if (mount) + { + mount->flags1.bits.ridden = false; + mount->flags4.bits.carrying_somebody = false; + // rebuild the ridden flags; mothers can carry more than one baby. + for (auto rider : world->units.other[units_other_id::ANY_RIDER]) + if (rider->relationship_ids[unit_relationship_type::RiderMount] == mount->id) + if (rider->profession == df::profession::BABY) + mount->flags4.bits.carrying_somebody = true; + else + mount->flags1.bits.ridden = true; + } + } + // If the unit was teleported out of a cage, break those links. if (auto cage = Units::getContainer(unit)) { From a863ed7695c50f200fcfd8392f0a1aad2db2f89a Mon Sep 17 00:00:00 2001 From: SilasD Date: Thu, 29 Oct 2026 19:37:05 -0700 Subject: [PATCH 24/42] Units.cpp teleport: don't break link between wagons and riders. also pull out "is a wagon" tests into predicates. --- library/modules/Units.cpp | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/library/modules/Units.cpp b/library/modules/Units.cpp index 8addee7e43..1a41b1b307 100644 --- a/library/modules/Units.cpp +++ b/library/modules/Units.cpp @@ -788,14 +788,25 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) if (!old_occ || !new_occ) return false; + auto is_wagon = [](df::unit *unit) { + if (!unit) + return false; + auto craw = df::creature_raw::find(unit->race); + return craw && craw->flags.is_set(df::creature_raw_flags::EQUIPMENT_WAGON); + }; + + auto uid_is_wagon = [&](int32_t unit_id) { + return is_wagon(df::unit::find(unit_id)); + }; + // EQUIPMENT units (e.g. wagons) occupy a 3x3 footprint centered on their // position; all other units occupy just their position tile - auto unit_extent = [](df::unit *u) { - if (auto craw = df::creature_raw::find(u->race); craw && - craw->flags.is_set(df::creature_raw_flags::EQUIPMENT_WAGON)) + auto unit_extent = [&](df::unit *unit) { + if (is_wagon(unit)) return 1; return 0; }; + const int extent = unit_extent(unit); auto for_each_occupied_tile = [&](df::coord center, auto &&fn) { @@ -868,20 +879,19 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) if (rider->relationship_ids[unit_relationship_type::RiderMount] == unit->id) // Wagon riders need special handling -- they are not at the center of the wagon, // and the draft animal riders are not even in the extent of the wagon. - if (auto craw = df::creature_raw::find(unit->race); craw && - craw->flags.is_set(df::creature_raw_flags::EQUIPMENT_WAGON)) + if (is_wagon(unit)) Units::teleport(rider, (rider->pos - src_pos) + unit->pos); else rider->pos = unit->pos; - // if the unit itself was a rider, break that link. + // if the unit itself was a rider, break that link (except do nothing to wagon-riders). if (unit->flags1.bits.rider) { auto mount = df::unit::find(unit->relationship_ids[df::unit_relationship_type::RiderMount]); - unit->flags1.bits.rider = false; - unit->relationship_ids[df::unit_relationship_type::RiderMount] = -1; - if (mount) + if (mount && !is_wagon(mount)) { + unit->flags1.bits.rider = false; + unit->relationship_ids[df::unit_relationship_type::RiderMount] = -1; mount->flags1.bits.ridden = false; mount->flags4.bits.carrying_somebody = false; // rebuild the ridden flags; mothers can carry more than one baby. From 6fdc04f1945b6749113efbfa6a0848f290232a97 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Fri, 30 Oct 2026 19:43:31 -0500 Subject: [PATCH 25/42] labormanager: respect unmanaged labors when probing fixes #6028 --- docs/changelog.txt | 1 + plugins/autolabor/modernengine.cpp | 26 +++++++++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/changelog.txt b/docs/changelog.txt index c2470dbe87..8d06b27363 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -60,6 +60,7 @@ Template for new versions: ## New Features ## Fixes +- `labormanager`: the modern engine's labor-mismatch probe no longer enables unmanaged labors on idle units or specialists, and no longer re-probes postings that can never be claimed (#6028) ## Misc Improvements diff --git a/plugins/autolabor/modernengine.cpp b/plugins/autolabor/modernengine.cpp index 38fae737e6..0fc7af63e2 100644 --- a/plugins/autolabor/modernengine.cpp +++ b/plugins/autolabor/modernengine.cpp @@ -267,6 +267,9 @@ struct LaborEscalation { std::map> writes; // unit -> labors we set }; static std::map labor_escalations; +// job ids that already completed a probe round; a posting that can never +// be claimed would otherwise re-escalate every cycle and probe forever +static std::set probed_jobs; static std::map labor_work; // live postings per labor, claimed or not // tool-detail membership as of the end of the last applied cycle; diffs // flag assignments made outside our reconcile calls (game or player) @@ -504,6 +507,7 @@ void ModernEngine::enable(color_ostream &out) for (auto &kv : labor_escalations) revert_escalation(kv.second); labor_escalations.clear(); + probed_jobs.clear(); unit_track.clear(); cycle_timestamp = 0; stat_starving = 0; @@ -538,6 +542,7 @@ void ModernEngine::map_unload() unclaimed_job.clear(); unclaimed_age.clear(); labor_escalations.clear(); + probed_jobs.clear(); unit_track.clear(); stat_starving = 0; stat_oldest_wait = 0; @@ -1769,6 +1774,10 @@ void ModernEngine::update(color_ostream &out) j->job_type == df::job_type::PlantSeeds || j->job_type == df::job_type::HarvestPlants) continue; + // a posting that already finished a probe round can never be + // claimed; starting another escalation for it would probe forever + if (probed_jobs.count(j->id)) + continue; LaborEscalation esc; esc.job_id = j->id; esc.jtype = j->job_type; @@ -1776,8 +1785,10 @@ void ModernEngine::update(color_ostream &out) // test the mapped labor on an idle dwarf first: distinguishes // "wrong labor" from "right labor, but every holder was busy" esc.candidates.push_back(l); + // the disputed set follows; unmanaged labors belong to the player's + // own work details, so the probe must not touch them for (auto m : disputed_labors) - if (m != l && !is_exclusive_labor(m)) + if (m != l && managed_labor_cache[m] && !is_exclusive_labor(m)) esc.candidates.push_back(m); INFO(labor_probe).print( "possible labor mapping mismatch: posting {} ({}) maps to {} " @@ -1822,8 +1833,12 @@ void ModernEngine::update(color_ostream &out) { if (auto u = df::unit::find(wi->first)) { - for (auto m : wi->second) - u->status.labors[m] = true; + // a unit that already claimed a job can't take the + // target posting anyway; re-asserting only lets it + // chain more work its details don't cover + if (!u->job.current_job) + for (auto m : wi->second) + u->status.labors[m] = true; ++wi; } else @@ -1832,9 +1847,12 @@ void ModernEngine::update(color_ostream &out) if (esc.next < esc.candidates.size()) { + // specialists are pinned to a skill detail; probing one + // would enable work its detail is meant to forbid df::unit *idle = nullptr; for (auto &info : units) if (info.state == IDLE && !info.excluded && + !specialist_ids.count(info.u->id) && !esc.writes.count(info.u->id)) { idle = info.u; @@ -1864,6 +1882,7 @@ void ModernEngine::update(color_ostream &out) "the probed set\n", esc.job_id, esc.job_name, ENUM_KEY_STR(unit_labor, l)); + probed_jobs.insert(esc.job_id); revert_escalation(esc); it = labor_escalations.erase(it); } @@ -1951,6 +1970,7 @@ void ModernEngine::update(color_ostream &out) ENUM_KEY_STR(unit_labor, l)); } + probed_jobs.insert(esc.job_id); revert_escalation(esc); it = labor_escalations.erase(it); } From 814eea8e1855e58cf1364ea06ff7e88d84e1f533 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sat, 31 Oct 2026 13:29:07 -0500 Subject: [PATCH 26/42] add `dfhack.timeout_until` callback triggered on a specific tick --- docs/changelog.txt | 1 + docs/dev/Lua API.rst | 9 +++++++++ library/LuaTools.cpp | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/docs/changelog.txt b/docs/changelog.txt index c2470dbe87..011579b80b 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -69,6 +69,7 @@ Template for new versions: ## API ## Lua +- ``dfhack.timeout_until``: new function to schedule a callback to run on a specified in-game tick ## Removed diff --git a/docs/dev/Lua API.rst b/docs/dev/Lua API.rst index 2f55e17184..9ef2f50ec5 100644 --- a/docs/dev/Lua API.rst +++ b/docs/dev/Lua API.rst @@ -3824,6 +3824,15 @@ Core context specific functions: Returns the timer id, or *nil* if unsuccessful due to world being unloaded. +* ``dfhack.timeout_until(tick,callback)`` + + Arranges for the callback to be called once the in-game + tick count (``df.global.world.frame_counter``) reaches + the specified value. The timer is canceled when the + world is unloaded, and cannot be queued until it is + loaded again. Returns the timer id, or *nil* if + unsuccessful due to world being unloaded. + * ``dfhack.timeout_active(id[,new_callback])`` Returns the active callback with the given id, or *nil* diff --git a/library/LuaTools.cpp b/library/LuaTools.cpp index 3b4bf16fe9..44fe872cb8 100644 --- a/library/LuaTools.cpp +++ b/library/LuaTools.cpp @@ -2036,6 +2036,38 @@ int dfhack_timeout(lua_State *L) return 1; } +int dfhack_timeout_until(lua_State *L) +{ + using df::global::world; + + // Parse arguments + lua_Number tick = luaL_checknumber(L, 1); + luaL_checktype(L, 2, LUA_TFUNCTION); + lua_settop(L, 2); + + if (!Core::getInstance().isWorldLoaded()) + { + lua_pushnil(L); + return 1; + } + + int delta = tick - world->frame_counter; + + if (delta <= 0) + luaL_error(L, "Invalid timeout: %d", delta); + + // Queue the timeout + int id = next_timeout_id++; + tick_timers.insert(std::pair(tick, id)); + + lua_rawgetp(L, LUA_REGISTRYINDEX, &DFHACK_TIMEOUTS_TOKEN); + lua_swap(L); + lua_rawseti(L, -2, id); + + lua_pushinteger(L, id); + return 1; +} + int dfhack_timeout_active(lua_State *L) { int id = luaL_optint(L, 1, -1); @@ -2147,6 +2179,8 @@ static void Lua::Core::InitCoreContext(color_ostream &out) lua_pushcfunction(State, dfhack_timeout); lua_setfield(State, -2, "timeout"); + lua_pushcfunction(State, dfhack_timeout_until); + lua_setfield(State, -2, "timeout_until"); lua_pushcfunction(State, dfhack_timeout_active); lua_setfield(State, -2, "timeout_active"); From 1662e63480d47d60b87bc9325fa302473eb454bb Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sat, 31 Oct 2026 14:49:28 -0500 Subject: [PATCH 27/42] eventful: correctly monitor harvestplant jobs `HarvestPlant` jobs don't use `completion_timer`, they instead use a unit action timer for the harvest job --- docs/changelog.txt | 1 + library/modules/EventManager.cpp | 73 +++++++++++++++++++++++++++++--- 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/docs/changelog.txt b/docs/changelog.txt index c2470dbe87..01133e8bfb 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -62,6 +62,7 @@ Template for new versions: ## Fixes ## Misc Improvements +- ``eventful``: ``onJobCompleted`` now fires for jobs that are deleted the same tick they complete (e.g. ``HarvestPlants``), detected via the worker's Job unit action timer ## Documentation - ``export-world-map``: added more format conversion examples. diff --git a/library/modules/EventManager.cpp b/library/modules/EventManager.cpp index 12aa4e7ed5..2a5f678a0b 100644 --- a/library/modules/EventManager.cpp +++ b/library/modules/EventManager.cpp @@ -31,6 +31,7 @@ #include "df/report.h" #include "df/plotinfost.h" #include "df/unit.h" +#include "df/unit_action.h" #include "df/unit_flags1.h" #include "df/unit_inventory_item.h" #include "df/unit_report_type.h" @@ -224,12 +225,28 @@ static std::vector startedJobs; struct JobCompleteData { int32_t id; int32_t completion_timer; - uint32_t flags_bits_repeat; + df::job_flags flags; }; +/* + * Some jobs are driven by the worker's Job unit action timer rather than by + * job.completion_timer, which stays -1 for the job's entire lifetime (e.g. + * HarvestPlants). DF deletes these jobs in the same tick the action expires, + * so they are never observed with completion_timer == 0. Snapshot the worker's + * action timer for claimed jobs that are in their work phase but whose + * completion_timer isn't counting down; when such a job disappears, a timer + * that was within reach of expiring during the elapsed ticks indicates a + * completion rather than a cancellation. + */ +struct ActionDrivenJobData { + int32_t unit_id; + int32_t unit_index; // cache for Units::get_cached_unit_by_global_id + int32_t action_timer; // -1 = worker's Job action not observed yet +}; static std::unordered_map seenJobs; static std::vector prevJobs; +static std::unordered_map actionDrivenJobs; //active units static unordered_set activeUnits; @@ -312,6 +329,7 @@ void DFHack::EventManager::onStateChange(color_ostream& out, state_change_event startedJobs.clear(); seenJobs.clear(); prevJobs.clear(); + actionDrivenJobs.clear(); tickQueue.clear(); livingUnits.clear(); buildings.clear(); @@ -560,10 +578,34 @@ static void manageJobCompletedEvent(color_ostream& out) { || (seenJob->general_refs.size() != job.general_refs.size())) { seenIt->second = Job::JobUniquePtr(Job::cloneJobStruct(&job, true)); } - } else if (job.completion_timer != -1) { + } else if (job.completion_timer != -1 || job.flags.bits.working) { // Restrict additions to seenJobs to jobs that we know have started. seenJobs.emplace(job.id, Job::JobUniquePtr(Job::cloneJobStruct(&job, true))); } + + if (job.completion_timer == -1 && job.flags.bits.working) { + auto ait = actionDrivenJobs.find(job.id); + if (ait == actionDrivenJobs.end()) { + if (df::unit* worker = Job::getWorker(&job)) + ait = actionDrivenJobs.emplace(job.id, + ActionDrivenJobData{worker->id, -1, -1}).first; + } + if (ait != actionDrivenJobs.end()) { + auto& data = ait->second; + df::unit* unit = Units::get_cached_unit_by_global_id(data.unit_id, data.unit_index); + if (!unit || unit->job.current_job != &job) { + // worker lost or swapped; re-register on a later poll + actionDrivenJobs.erase(ait); + } else { + for (auto action : unit->actions) { + if (action->type == unit_action_type::Job) { + data.action_timer = action->data.job.timer; + break; + } + } + } + } + } /* * We still need to push back all jobs to maintain the invariant of the * algorithm used with prevJobs and nowJobs. @@ -590,7 +632,7 @@ static void manageJobCompletedEvent(color_ostream& out) { * 1 in nowJobs is less than the smallest ID in prevJobs, * which breaks the algorithm. */ - nowJobs.emplace_back(job.id, job.completion_timer, (bool)job.flags.bits.repeat); + nowJobs.emplace_back(job.id, job.completion_timer, job.flags); } // Do we want this check? @@ -663,6 +705,7 @@ static void manageJobCompletedEvent(color_ostream& out) { auto prevIt = prevJobs.begin(); auto nowIt = nowJobs.begin(); + int32_t tick = df::global::world->frame_counter; /* * Iterate through two ordered sets, prevJobs and nowJobs, where job IDs in nowJobs are invariably * greater than or equal to job IDs in prevJobs. The algorithm maintains the invariant that for each @@ -674,7 +717,27 @@ static void manageJobCompletedEvent(color_ostream& out) { auto& prevJob = *prevIt; if (nowIt == nowJobs.end() || prevJob.id != nowIt->id) { // job ID is in prevJobs. ID does not exist in nowJobs. // recently finished or cancelled job - if (!prevJob.flags_bits_repeat && prevJob.completion_timer == 0) { + bool completed = prevJob.completion_timer == 0; + if (!completed) { + /* + * Jobs whose work is tracked by the worker's Job unit action + * are deleted the same tick they complete, before + * completion_timer ever reaches 0. If the last observed action + * timer was close enough to expiring that it could have hit 0 + * during the elapsed ticks, count it as a completion. + */ + auto ait = actionDrivenJobs.find(prevJob.id); + if (ait != actionDrivenJobs.end() + && prevJob.flags.bits.working + && !prevJob.flags.bits.suspend + && !prevJob.flags.bits.item_lost + && ait->second.action_timer >= 0 + && ait->second.action_timer <= tick - eventLastTick[EventType::JOB_COMPLETED]) { + completed = true; + } + } + actionDrivenJobs.erase(prevJob.id); + if (completed && !prevJob.flags.bits.repeat) { // It should be in seenJobs. auto seenIt = seenJobs.find(prevJob.id); if (seenIt != seenJobs.end()) { @@ -688,7 +751,7 @@ static void manageJobCompletedEvent(color_ostream& out) { } } else { // prevIt job ID and nowIt job ID are equal. // could have just finished if it's a repeat job - if (prevJob.flags_bits_repeat && prevJob.completion_timer == 0 + if (prevJob.flags.bits.repeat && prevJob.completion_timer == 0 && nowIt->completion_timer == -1) { // It should be in seenJobs. auto seenIt = seenJobs.find(prevJob.id); From a33f7ed8631c94a4963d47819ad26aa29e28e2d3 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sat, 31 Oct 2026 17:31:49 -0500 Subject: [PATCH 28/42] autofarm improvements add ability to enable/disable individual farms add a farm overlay a companion script PR adds a configuration GUI --- docs/changelog.txt | 2 + docs/plugins/autofarm.rst | 8 ++ plugins/CMakeLists.txt | 2 +- plugins/autofarm.cpp | 281 ++++++++++++++++++++++++++++++++------ plugins/lua/autofarm.lua | 184 +++++++++++++++++++++++++ test/plugins/autofarm.lua | 17 +++ 6 files changed, 448 insertions(+), 46 deletions(-) create mode 100644 plugins/lua/autofarm.lua diff --git a/docs/changelog.txt b/docs/changelog.txt index c2470dbe87..34a62cde13 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -58,8 +58,10 @@ Template for new versions: ## New Tools ## New Features +- `autofarm`: farm plots can now be individually excluded from autofarm management via new controls on the farm plot information sheet, which also warn when a farm's crops are being managed by autofarm ## Fixes +- `autofarm`: farms that span multiple biomes are now handled correctly: crops that can grow in any of the farm's biomes can be assigned to the farm, instead of only using the biome of the farm's center tile ## Misc Improvements diff --git a/docs/plugins/autofarm.rst b/docs/plugins/autofarm.rst index 0fe98f2b8b..ee572bea41 100644 --- a/docs/plugins/autofarm.rst +++ b/docs/plugins/autofarm.rst @@ -9,6 +9,14 @@ Periodically scan your plant stocks and assign crops to your farm plots based on which plant stocks are low (as long as you have the appropriate seeds). The target threshold for each crop type is configurable. +Farms that span multiple biomes are handled correctly: any crop that can grow +in any of the biomes covered by the farm's tiles may be assigned to that farm. + +Individual farm plots can be excluded from autofarm management via the controls +that appear on the farm plot information sheet. That sheet also provides a +shortcut to `gui/autofarm`, a convenient UI for setting crop thresholds and +enabling or disabling autofarm. + Usage ----- diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 45290fbc3c..f8a75436bd 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -51,7 +51,7 @@ if(BUILD_SUPPORTED) dfhack_plugin(autoclothing autoclothing.cpp LINK_LIBRARIES lua) dfhack_plugin(design design.cpp LINK_LIBRARIES lua) dfhack_plugin(autodump autodump.cpp) - dfhack_plugin(autofarm autofarm.cpp) + dfhack_plugin(autofarm autofarm.cpp LINK_LIBRARIES lua) #dfhack_plugin(autogems autogems.cpp LINK_LIBRARIES jsoncpp_static) add_subdirectory(autolabor) dfhack_plugin(autonestbox autonestbox.cpp LINK_LIBRARIES lua) diff --git a/plugins/autofarm.cpp b/plugins/autofarm.cpp index 65a08123d9..3e51dc8bce 100644 --- a/plugins/autofarm.cpp +++ b/plugins/autofarm.cpp @@ -1,6 +1,9 @@ #include "Debug.h" +#include "LuaTools.h" #include "PluginManager.h" +#include "PluginLua.h" +#include "modules/Buildings.h" #include "modules/Items.h" #include "modules/Maps.h" #include "modules/World.h" @@ -22,8 +25,8 @@ #include "df/unit.h" #include "df/world.h" +#include #include -#include using namespace DFHack; using namespace df::enums; @@ -72,6 +75,26 @@ class AutoFarm { return (thresholds.count(id) > 0) ? thresholds[id] : defaultThreshold; } + void removeThreshold(int id) + { + thresholds.erase(id); + } + + int getDefault() + { + return defaultThreshold; + } + + const std::map& getThresholds() + { + return thresholds; + } + + const std::map& getLastCounts() + { + return lastCounts; + } + void setDefault(int val) { defaultThreshold = val; @@ -194,6 +217,82 @@ class AutoFarm { return raw ? raw->name : "NONE"; } + // gathers the set of biomes covered by the farm's tiles, since a farm may + // span multiple biomes and DF allows planting any crop that can grow in + // any of them (in the tiles of the respective biome) + std::set get_farm_biomes(df::building_farmplotst* farm) + { + std::set biomes; + for (int32_t x = farm->x1; x <= farm->x2; x++) + { + for (int32_t y = farm->y1; y <= farm->y2; y++) + { + if (!Buildings::containsTile(farm, df::coord2d(x, y))) + continue; + auto des = Maps::getTileDesignation(x, y, farm->z); + if (!des) + continue; + if (des->bits.subterranean) + { + // any subterranean tile can grow any crop that grows + // underground + biomes.insert(df::biome_type::SUBTERRANEAN_WATER); + biomes.insert(df::biome_type::SUBTERRANEAN_CHASM); + biomes.insert(df::biome_type::SUBTERRANEAN_LAVA); + } + else if (Maps::getRegionBiome(Maps::getTileBiomeRgn(df::coord(x, y, farm->z)))) + { + df::coord2d region(Maps::getTileBiomeRgn(df::coord(x, y, farm->z))); + biomes.insert(Maps::getBiomeType(region.x, region.y)); + } + } + } + return biomes; + } + +public: + bool is_farm_excluded(int32_t farm_id) + { + auto item = World::GetPersistentSiteData("autofarm/excluded_farm/" + std::to_string(farm_id)); + return item.isValid() && item.ival(0) != 0; + } + + void set_farm_excluded(int32_t farm_id, bool excluded) + { + const std::string key = "autofarm/excluded_farm/" + std::to_string(farm_id); + auto item = World::GetPersistentSiteData(key); + if (excluded) + { + if (!item.isValid()) + item = World::AddPersistentSiteData(key); + if (item.isValid()) + item.ival(0) = 1; + } + else if (item.isValid()) + { + World::DeletePersistentData(item); + } + } + + std::vector get_excluded_farms() + { + std::vector items; + std::vector ret; + World::GetPersistentSiteData(&items, "autofarm/excluded_farm/", true); + for (auto& i : items) + { + if (!i.isValid() || i.ival(0) == 0) + continue; + int32_t farm_id = atoi(i.key().substr(strlen("autofarm/excluded_farm/")).c_str()); + auto bld = df::building::find(farm_id); + // ignore entries left behind by destroyed farm plots + if (bld && bld->getType() == df::building_type::FarmPlot) + ret.push_back(farm_id); + } + return ret; + } + +private: void set_farm(color_ostream& out, int new_plant_id, df::building_farmplotst* farm, int season) { int old_plant_id = farm->plant_id[season]; @@ -206,54 +305,100 @@ class AutoFarm { } } - void set_farms(color_ostream& out, const std::set& plants, const std::vector& farms) +public: + void set_farms(color_ostream& out, const std::map>& plants, + const std::vector>>& farms) { - // this algorithm attempts to change as few farms as possible, while ensuring that - // the number of farms planting each eligible plant is "as equal as possible" + // this algorithm attempts to change as few farms as possible, while + // ensuring that the number of farms planting each plant is "as equal + // as possible". unlike previously, each farm may have a different set + // of plants that it can grow, since a farm can span multiple biomes int season = *df::global::cur_season; - if (farms.empty() || plants.empty()) + // the plants that each farm is eligible to grow, and the list of + // farms assigned to each plant + std::map> eligible; + std::map> assigned; + std::map counters; + std::vector unassigned; + + for (auto& [farm, biomes] : farms) { - // if no more plants were requested, fallow all farms - // if there were no farms, do nothing - for (auto farm : farms) + for (auto biome : biomes) { - set_farm(out, -1, farm, season); + auto it = plants.find(biome); + if (it != plants.end()) + eligible[farm].insert(it->second.begin(), it->second.end()); } - return; + for (auto plant : eligible[farm]) + assigned[plant]; // ensure every eligible plant can receive farms } - int min = farms.size() / plants.size(); // the number of farms that should plant each eligible plant, rounded down - int extra = farms.size() - min * plants.size(); // the remainder that cannot be evenly divided + // keep the current assignment for farms whose crop is still eligible + for (auto& [farm, biomes] : farms) + { + int cur = farm->plant_id[season]; + if (cur >= 0 && eligible[farm].count(cur)) + { + assigned[cur].push_back(farm); + counters[cur]++; + } + else + { + unassigned.push_back(farm); + } + } - std::map counters; - std::queue toChange; + // assign farms with the fewest options first so that constrained + // farms don't get crowded out by more flexible ones + std::sort(unassigned.begin(), unassigned.end(), + [&](df::building_farmplotst* a, df::building_farmplotst* b) + { + return eligible[a].size() < eligible[b].size(); + }); - for (auto farm : farms) + for (auto farm : unassigned) { - int o = farm->plant_id[season]; - if (plants.count(o) == 0 || counters[o] > min || (counters[o] == min && extra == 0)) - toChange.push(farm); // this farm is an excess instance for the plant it is currently planting - else + if (eligible[farm].empty()) { - if (counters[o] == min) - extra--; // allocate off one of the remainder farms - counters[o]++; + set_farm(out, -1, farm, season); + continue; } + int best = *std::min_element(eligible[farm].begin(), eligible[farm].end(), + [&](int a, int b) { return counters[a] < counters[b]; }); + set_farm(out, best, farm, season); + assigned[best].push_back(farm); + counters[best]++; } - for (auto n : plants) + // rebalance: move farms from overrepresented plants to + // underrepresented ones while their counts differ by more than one. + // each move strictly decreases the variance, so this terminates + for (bool moved = true; moved;) { - int c = counters[n]; - while (toChange.size() > 0 && (c < min || (c == min && extra > 0))) + moved = false; + for (auto& [src, src_farms] : assigned) { - // pick one of the excess farms and change it to plant this plant - df::building_farmplotst* farm = toChange.front(); - set_farm(out, n, farm, season); - toChange.pop(); - if (c++ == min) - extra--; + for (auto& [dst, dst_farms] : assigned) + { + if (counters[src] <= counters[dst] + 1) + continue; + auto it = std::find_if(src_farms.begin(), src_farms.end(), + [&](df::building_farmplotst* f) { return eligible[f].count(dst); }); + if (it == src_farms.end()) + continue; + auto farm = *it; + set_farm(out, dst, farm, season); + src_farms.erase(it); + dst_farms.push_back(farm); + counters[src]--; + counters[dst]++; + moved = true; + break; + } + if (moved) + break; } } } @@ -303,28 +448,18 @@ class AutoFarm { } } - std::map> farms; + std::vector>> farms; for (auto& bb : world->buildings.other[df::buildings_other_id::FARM_PLOT]) { auto farm = virtual_cast(bb); - if (farm->flags.bits.exists) + if (farm->flags.bits.exists && !is_farm_excluded(farm->id)) { - df::biome_type biome; - if (Maps::getTileDesignation(bb->centerx, bb->centery, bb->z)->bits.subterranean) - biome = biome_type::SUBTERRANEAN_WATER; - else { - df::coord2d region(Maps::getTileBiomeRgn(df::coord(bb->centerx, bb->centery, bb->z))); - biome = Maps::getBiomeType(region.x, region.y); - } - farms[biome].push_back(farm); + farms.emplace_back(farm, get_farm_biomes(farm)); } } - for (auto& ff : farms) - { - set_farms(out, plants[ff.first], ff.second); - } + set_farms(out, plants, farms); out << std::flush; } @@ -545,3 +680,59 @@ static command_result autofarm(color_ostream& out, std::vector& par return CR_OK; } + +static void autofarm_setDefaultThreshold(color_ostream& out, int32_t val) +{ + autofarmInstance->setDefault(val); + autofarmInstance->save_state(out); +} + +static void autofarm_setThreshold(color_ostream& out, int32_t plant_index, int32_t val) +{ + autofarmInstance->setThreshold(plant_index, val); + autofarmInstance->save_state(out); +} + +static void autofarm_removeThreshold(color_ostream& out, int32_t plant_index) +{ + autofarmInstance->removeThreshold(plant_index); + autofarmInstance->save_state(out); +} + +static bool autofarm_isFarmExcluded(int32_t farm_id) +{ + return autofarmInstance->is_farm_excluded(farm_id); +} + +static void autofarm_setFarmExcluded(color_ostream& out, int32_t farm_id, bool excluded) +{ + autofarmInstance->set_farm_excluded(farm_id, excluded); +} + +static int autofarm_getData(lua_State* L) +{ + lua_newtable(L); + int ctable = lua_gettop(L); + Lua::SetField(L, autofarmInstance->getDefault(), ctable, "default_threshold"); + Lua::Push(L, autofarmInstance->getThresholds()); + lua_setfield(L, ctable, "thresholds"); + Lua::Push(L, autofarmInstance->getLastCounts()); + lua_setfield(L, ctable, "last_counts"); + Lua::Push(L, autofarmInstance->get_excluded_farms()); + lua_setfield(L, ctable, "excluded_farms"); + return 1; +} + +DFHACK_PLUGIN_LUA_FUNCTIONS { + DFHACK_LUA_FUNCTION(autofarm_setDefaultThreshold), + DFHACK_LUA_FUNCTION(autofarm_setThreshold), + DFHACK_LUA_FUNCTION(autofarm_removeThreshold), + DFHACK_LUA_FUNCTION(autofarm_isFarmExcluded), + DFHACK_LUA_FUNCTION(autofarm_setFarmExcluded), + DFHACK_LUA_END +}; + +DFHACK_PLUGIN_LUA_COMMANDS { + DFHACK_LUA_COMMAND(autofarm_getData), + DFHACK_LUA_END +}; diff --git a/plugins/lua/autofarm.lua b/plugins/lua/autofarm.lua new file mode 100644 index 0000000000..590e46fb9a --- /dev/null +++ b/plugins/lua/autofarm.lua @@ -0,0 +1,184 @@ +local _ENV = mkmodule('plugins.autofarm') + +local gui = require('gui') +local overlay = require('plugins.overlay') +local widgets = require('gui.widgets') + +local function get_farm() + local bld = dfhack.gui.getSelectedBuilding(true) + if bld and bld:getType() == df.building_type.FarmPlot then + return bld + end +end + +local function farm_is_managed(farm) + return farm and autofarm_isFarmExcluded and + not autofarm_isFarmExcluded(farm.id) +end + +-------------------------------- +-- FarmControlsOverlay +-- +-- toggle strip shown on the farm plot info sheet: enables/disables autofarm +-- globally, marks the inspected farm as managed or manually controlled, and +-- provides a link to the autofarm configuration GUI + +FarmControlsOverlay = defclass(FarmControlsOverlay, overlay.OverlayWidget) +FarmControlsOverlay.ATTRS{ + desc='Adds autofarm controls to the farm plot info sheet.', + default_pos={x=-43, y=13}, + version=3, + default_enabled=true, + viewscreens='dwarfmode/ViewSheets/BUILDING/FarmPlot', + frame={w=50, h=2}, + frame_background=gui.CLEAR_PEN, +} + +function FarmControlsOverlay:init() + self:addviews{ + widgets.ToggleHotkeyLabel{ + view_id='autofarm_enabled', + frame={t=0, l=0}, + label='autofarm is', + key='CUSTOM_CTRL_A', + options={{value=true, label='enabled', pen=COLOR_GREEN}, + {value=false, label='disabled', pen=COLOR_RED}}, + on_change=function(val) setEnabled(val) end, + }, + widgets.ToggleHotkeyLabel{ + view_id='farm_managed', + frame={t=1, l=0}, + label='this farm is', + key='CUSTOM_CTRL_M', + enabled=function() return isEnabled() end, + options={{value=true, label='managed by autofarm', pen=COLOR_GREEN}, + {value=false, label='managed manually', pen=COLOR_YELLOW}}, + on_change=function(val) + local farm = get_farm() + if farm then autofarm_setFarmExcluded(farm.id, not val) end + end, + }, + widgets.TextButton{ + frame={t=0, r=0, w=18, h=1}, + label='configure', + key='CUSTOM_CTRL_S', + on_activate=function() dfhack.run_script('gui/autofarm') end, + }, + } +end + +function FarmControlsOverlay:render(dc) + self.subviews.autofarm_enabled:setOption(isEnabled()) + self.subviews.farm_managed:setOption(farm_is_managed(get_farm())) + FarmControlsOverlay.super.render(self, dc) +end + +-------------------------------- +-- FarmCoverOverlay +-- +-- opaque panel shown over the crop list of farms managed by autofarm. warns +-- the player that manual crop selection is overridden by autofarm and +-- provides links to the autofarm configuration GUI and to per-farm control + +local function get_crop_name(plant_id) + if not plant_id or plant_id < 0 then return 'fallow' end + local raw = df.plant_raw.find(plant_id) + return raw and raw.name or 'unknown' +end + +FarmCoverOverlay = defclass(FarmCoverOverlay, overlay.OverlayWidget) +FarmCoverOverlay.ATTRS{ + desc='Covers the crop list of farm plots managed by autofarm.', + default_pos={x=-40, y=16}, + version=4, + default_enabled=true, + viewscreens='dwarfmode/ViewSheets/BUILDING/FarmPlot', + frame={w=57, h=22}, + frame_style=gui.FRAME_MEDIUM, + frame_background=gui.CLEAR_PEN, + visible=function() + local farm = get_farm() + return isEnabled() and farm_is_managed(farm) and + farm:getBuildStage() == farm:getMaxBuildStage() + end, +} + +function FarmCoverOverlay:init() + self:addviews{ + widgets.Label{ + frame={t=1, l=2}, + text='This farm is managed by autofarm', + text_pen=COLOR_LIGHTGREEN, + }, + widgets.Label{ + view_id='now_planting', + frame={t=2, l=2}, + text='Now planting: ?', + }, + widgets.WrappedLabel{ + frame={t=3, l=2, r=2}, + text_to_wrap='Crops are chosen automatically based on your'.. + ' configured stock thresholds. Manual crop selection'.. + ' will be overridden.', + }, + widgets.WrappedLabel{ + view_id='fallow_note', + frame={t=6, l=2, r=2}, + text_pen=COLOR_YELLOW, + }, + widgets.TextButton{ + frame={b=3, l=2, w=24, h=1}, + label='configure autofarm', + key='CUSTOM_CTRL_S', + on_activate=function() dfhack.run_script('gui/autofarm') end, + }, + widgets.TextButton{ + frame={b=3, l=30, w=30, h=1}, + label='manage this farm manually', + key='CUSTOM_CTRL_M', + on_activate=function() + local farm = get_farm() + if farm then autofarm_setFarmExcluded(farm.id, true) end + end, + }, + } +end + +function FarmCoverOverlay:render(dc) + local farm = get_farm() + if farm then + self.subviews.now_planting:setText{ + 'Now planting: ', + {text=get_crop_name(farm.plant_id[df.global.cur_season]), + pen=COLOR_LIGHTCYAN}, + } + local all_fallow = true + for i=0,3 do + if farm.plant_id[i] >= 0 then + all_fallow = false + break + end + end + self.subviews.fallow_note:setText(all_fallow and + 'All seasons are fallow: no plantable seeds available, or all'.. + ' crops are above their thresholds.' or '') + end + FarmCoverOverlay.super.render(self, dc) +end + +-- swallow mouse input over the covered area so that the underlying crop +-- list cannot be manipulated while autofarm is in control +function FarmCoverOverlay:onInput(keys) + if FarmCoverOverlay.super.onInput(self, keys) then return true end + if (keys._MOUSE_L or keys._MOUSE_R) and self:getMousePos() then + return true + end + return false +end + +OVERLAY_WIDGETS = { + farm_controls=FarmControlsOverlay, + farm_cover=FarmCoverOverlay, +} + +return _ENV diff --git a/test/plugins/autofarm.lua b/test/plugins/autofarm.lua index 0573d7379d..3936922101 100644 --- a/test/plugins/autofarm.lua +++ b/test/plugins/autofarm.lua @@ -38,3 +38,20 @@ function test.threshold_rejects_bad_plant() expect.true_(output:find('Cannot find plant', 1, true) ~= nil or status ~= 0) end + +function test.farm_exclusion() + local plugin = require('plugins.autofarm') + expect.false_(plugin.autofarm_isFarmExcluded(-1)) + plugin.autofarm_setFarmExcluded(-1, true) + expect.true_(plugin.autofarm_isFarmExcluded(-1)) + plugin.autofarm_setFarmExcluded(-1, false) + expect.false_(plugin.autofarm_isFarmExcluded(-1)) +end + +function test.get_data() + local data = require('plugins.autofarm').autofarm_getData() + expect.eq('number', type(data.default_threshold)) + expect.eq('table', type(data.thresholds)) + expect.eq('table', type(data.last_counts)) + expect.eq('table', type(data.excluded_farms)) +end From 29de10431ed7d9d4f6289e83617bf491c368468f Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sat, 31 Oct 2026 17:53:28 -0500 Subject: [PATCH 29/42] pin scripts for the PR so the doc builds can go --- scripts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts b/scripts index 4c4f5bf510..2bcaa138db 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 4c4f5bf510e22fb96857328638e93ded3f608d0e +Subproject commit 2bcaa138dbd20772e94039d78e8a62850b01e100 From 081b82ac1dc094df21c9089604ad92ba719ee727 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Sat, 31 Oct 2026 18:53:50 -0500 Subject: [PATCH 30/42] phase two: improved growth support --- docs/changelog.txt | 1 + docs/plugins/autofarm.rst | 10 ++ plugins/autofarm.cpp | 213 +++++++++++++++++++++++++++++++++++--- scripts | 2 +- test/plugins/autofarm.lua | 47 +++++++++ 5 files changed, 260 insertions(+), 13 deletions(-) diff --git a/docs/changelog.txt b/docs/changelog.txt index 34a62cde13..20fa29681c 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -59,6 +59,7 @@ Template for new versions: ## New Features - `autofarm`: farm plots can now be individually excluded from autofarm management via new controls on the farm plot information sheet, which also warn when a farm's crops are being managed by autofarm +- `autofarm`: thresholds can now be assigned to specific growth products of a plant (e.g. ``BUSH_QUARRY:LEAVES``), so crops that are grown for their growth items can be farmed to a target amount of that product ## Fixes - `autofarm`: farms that span multiple biomes are now handled correctly: crops that can grow in any of the farm's biomes can be assigned to the farm, instead of only using the biome of the farm's center tile diff --git a/docs/plugins/autofarm.rst b/docs/plugins/autofarm.rst index ee572bea41..0d070eeedc 100644 --- a/docs/plugins/autofarm.rst +++ b/docs/plugins/autofarm.rst @@ -34,6 +34,13 @@ Usage You can find the identifiers for the crop types in your world by running the following command: ``getplants -f`` +Some plants produce usable growth items (such as fruits, leaves, or bulbs) +when processed or gathered. These products normally count toward the plant's +threshold together with the plant items themselves, but you can also assign a +threshold to a specific growth product with the ``:`` syntax. +A plant is grown while either its pooled stock or any of its assigned growth +products is below the corresponding threshold. + Examples -------- @@ -41,3 +48,6 @@ Examples Set the default threshold to 30. ``autofarm threshold 150 MUSHROOM_HELMET_PLUMP GRASS_TAIL_PIG`` Set the threshold for Plump Helmets and Pig Tails to 150 +``autofarm threshold 30 BUSH_QUARRY:LEAVES`` + Keep at least 30 Quarry Bush Leaves in stock, growing Quarry Bushes until + you have enough of them to process diff --git a/plugins/autofarm.cpp b/plugins/autofarm.cpp index 3e51dc8bce..2881bfcc96 100644 --- a/plugins/autofarm.cpp +++ b/plugins/autofarm.cpp @@ -6,6 +6,7 @@ #include "modules/Buildings.h" #include "modules/Items.h" #include "modules/Maps.h" +#include "modules/Materials.h" #include "modules/World.h" #include "df/biome_type.h" @@ -19,6 +20,9 @@ #include "df/item_plant_growthst.h" #include "df/item_seedsst.h" #include "df/items_other_id.h" +#include "df/material.h" +#include "df/material_flags.h" +#include "df/plant_growth.h" #include "df/plant_raw.h" #include "df/plant_raw_flags.h" #include "df/plotinfost.h" @@ -54,7 +58,15 @@ class AutoFarm { std::map thresholds; int defaultThreshold = 50; + // thresholds for specific growth products of a plant, keyed by + // (plant index, growth id). growth items whose growth has no explicit + // threshold count toward the plant's pooled threshold instead + std::map, int> growth_thresholds; + std::map lastCounts; + // stockpile growth item counts, keyed by (plant index, growth id). + // populated for reporting; untracked growths also count toward lastCounts + std::map, int> lastGrowthCounts; public: void initialize() @@ -62,7 +74,10 @@ class AutoFarm { thresholds.clear(); defaultThreshold = 50; + growth_thresholds.clear(); + lastCounts.clear(); + lastGrowthCounts.clear(); } void setThreshold(int id, int val) @@ -80,6 +95,26 @@ class AutoFarm { thresholds.erase(id); } + void setGrowthThreshold(int plant_id, const std::string& growth_id, int val) + { + growth_thresholds[std::make_pair(plant_id, growth_id)] = val; + } + + void removeGrowthThreshold(int plant_id, const std::string& growth_id) + { + growth_thresholds.erase(std::make_pair(plant_id, growth_id)); + } + + const std::map, int>& getGrowthThresholds() + { + return growth_thresholds; + } + + const std::map, int>& getLastGrowthCounts() + { + return lastGrowthCounts; + } + int getDefault() { return defaultThreshold; @@ -217,6 +252,34 @@ class AutoFarm { return raw ? raw->name : "NONE"; } + // whether the growth's material produces stockpilable growth items; + // only these are ever produced by processing jobs and gathering, so + // they are the only growths that can be meaningfully tracked + static bool is_stockpile_growth(df::plant_growth* growth) + { + MaterialInfo mat(growth->mat_type, growth->mat_index); + return mat.isValid() && + mat.material->flags.is_set(df::material_flags::STOCKPILE_PLANT_GROWTH); + } + + // the id of the stockpile growth at the given index, or "" if the + // growth doesn't exist or isn't stockpilable + static std::string stockpile_growth_id(df::plant_raw* plant, int32_t subtype) + { + if (subtype >= 0 && (size_t)subtype < plant->growths.size() && + is_stockpile_growth(plant->growths[subtype])) + return plant->growths[subtype]->id; + return ""; + } + + static bool has_stockpile_growth(df::plant_raw* plant, const std::string& growth_id) + { + for (auto g : plant->growths) + if (g->id == growth_id && is_stockpile_growth(g)) + return true; + return false; + } + // gathers the set of biomes covered by the farm's tiles, since a farm may // span multiple biomes and DF allows planting any crop that can grow in // any of them (in the tiles of the respective biome) @@ -422,7 +485,9 @@ class AutoFarm { // have to scan both items[PLANT] and items[PLANT_GROWTH] because agricultural products can be either - auto count = [&, this](df::item* i) { + lastGrowthCounts.clear(); + + auto count_plant = [&, this](df::item* i) { auto mat = i->getMaterialIndex(); if ((i->flags.whole & bad_flags) == 0 && plantable_plants.count(mat) > 0) @@ -431,17 +496,55 @@ class AutoFarm { } }; + auto count_growth = [&, this](df::item* i) { + auto g = virtual_cast(i); + auto mat = i->getMaterialIndex(); + if (!g || (i->flags.whole & bad_flags) != 0 || + plantable_plants.count(mat) == 0) + return; + df::plant_raw* plant = world->raws.plants.all[mat]; + const std::string gid = stockpile_growth_id(plant, g->subtype); + const auto key = std::make_pair(mat, gid); + if (!gid.empty() && growth_thresholds.count(key)) + { + // tracked growths count only toward their own threshold + lastGrowthCounts[key] += i->getStackSize(); + } + else + { + if (!gid.empty()) + // unconfigured stockpile growths are tracked + // separately for reporting, but still count toward + // the plant's pooled threshold + lastGrowthCounts[key] += i->getStackSize(); + lastCounts[mat] += i->getStackSize(); + } + }; + for (auto i : world->items.other[df::items_other_id::PLANT]) - count(i); + count_plant(i); for (auto i : world->items.other[df::items_other_id::PLANT_GROWTH]) - count(i); + count_growth(i); std::map> plants; for (auto& plantable : plantable_plants) { df::plant_raw* plant = world->raws.plants.all[plantable.first]; - if (lastCounts[plant->index] < getThreshold(plant->index)) + // a plant is in demand when its pooled stock is under threshold + // or any explicitly tracked growth of it is under threshold + bool under = lastCounts[plant->index] < getThreshold(plant->index); + if (!under) + for (auto& gt : growth_thresholds) + { + if (gt.first.first == plant->index && + lastGrowthCounts[gt.first] < gt.second) + { + under = true; + break; + } + } + if (under) for (auto biome : plantable.second) { plants[biome].insert(plant->index); @@ -481,6 +584,14 @@ class AutoFarm { auto plant = world->raws.plants.all[th.first]; out << plant->id << " limit " << getThreshold(th.first) << " current 0" << '\n'; } + + for (auto& gt : growth_thresholds) + { + auto plant = world->raws.plants.all[gt.first.first]; + out << plant->id << ':' << gt.first.second << " limit " << gt.second + << " current " << lastGrowthCounts[gt.first] << '\n'; + } + out << "Default: " << defaultThreshold << '\n'; out << std::flush; @@ -521,15 +632,32 @@ class AutoFarm { const auto allPlants = world->raws.plants.all; const std::string id = i.val(); const int val = i.ival(0); - const auto plant = std::find_if(std::begin(allPlants), std::end(allPlants), [id](df::plant_raw* p) { return p->id == id; }); - if (plant != std::end(allPlants)) + const auto colon = id.find(':'); + const std::string plant_id = id.substr(0, colon); + const auto plant = std::find_if(std::begin(allPlants), std::end(allPlants), [plant_id](df::plant_raw* p) { return p->id == plant_id; }); + if (plant == std::end(allPlants)) + { + WARN(control, out).print("threshold for unknown plant {} in saved configuration ignored\n", id); + } + else if (colon == std::string::npos) { setThreshold((*plant)->index, val); INFO(control, out).print("threshold of {} for plant {} in saved configuration loaded\n", val, id); } else { - WARN(control, out).print("threshold for unknown plant {} in saved configuration ignored\n", id); + // "PLANT:GROWTH" entries hold thresholds for specific + // stockpile growth products of the plant + const std::string growth_id = id.substr(colon + 1); + if (has_stockpile_growth(*plant, growth_id)) + { + setGrowthThreshold((*plant)->index, growth_id, val); + INFO(control, out).print("threshold of {} for growth {} in saved configuration loaded\n", val, id); + } + else + { + WARN(control, out).print("threshold for unknown growth {} in saved configuration ignored\n", id); + } } } } @@ -554,6 +682,15 @@ class AutoFarm { cfgThreshold.val() = plantID; cfgThreshold.ival(0) = t.second; } + + for (const auto& t : growth_thresholds) + { + const std::string productID = world->raws.plants.all[t.first.first]->id + ":" + t.first.second; + const std::string keyName = "autofarm/threshold/" + productID; + PersistentDataItem cfgThreshold = World::AddPersistentSiteData(keyName); + cfgThreshold.val() = productID; + cfgThreshold.ival(0) = t.second; + } } }; @@ -625,19 +762,29 @@ static command_result setThresholds(color_ostream& out, std::vector std::string id = parameters[i]; std::transform(id.begin(), id.end(), id.begin(), ::toupper); + // a threshold may apply to a plant ("PLANT") or to a specific + // stockpile growth product of a plant ("PLANT:GROWTH") + const auto colon = id.find(':'); + const std::string plant_id = id.substr(0, colon); + const std::string growth_id = colon == std::string::npos ? "" : id.substr(colon + 1); + bool ok = false; for (auto plant : world->raws.plants.all) { - if (plant->flags.is_set(df::plant_raw_flags::SEED) && (plant->id == id)) - { + if (!plant->flags.is_set(df::plant_raw_flags::SEED) || (plant->id != plant_id)) + continue; + if (growth_id.empty()) autofarmInstance->setThreshold(plant->index, val); - ok = true; + else if (AutoFarm::has_stockpile_growth(plant, growth_id)) + autofarmInstance->setGrowthThreshold(plant->index, growth_id, val); + else break; - } + ok = true; + break; } if (!ok) { - out << "Cannot find plant with id " << id << '\n' << std::flush; + out << "Cannot find plant or growth with id " << id << '\n' << std::flush; return CR_WRONG_USAGE; } } @@ -699,6 +846,18 @@ static void autofarm_removeThreshold(color_ostream& out, int32_t plant_index) autofarmInstance->save_state(out); } +static void autofarm_setGrowthThreshold(color_ostream& out, int32_t plant_index, std::string growth_id, int32_t val) +{ + autofarmInstance->setGrowthThreshold(plant_index, growth_id, val); + autofarmInstance->save_state(out); +} + +static void autofarm_removeGrowthThreshold(color_ostream& out, int32_t plant_index, std::string growth_id) +{ + autofarmInstance->removeGrowthThreshold(plant_index, growth_id); + autofarmInstance->save_state(out); +} + static bool autofarm_isFarmExcluded(int32_t farm_id) { return autofarmInstance->is_farm_excluded(farm_id); @@ -709,6 +868,30 @@ static void autofarm_setFarmExcluded(color_ostream& out, int32_t farm_id, bool e autofarmInstance->set_farm_excluded(farm_id, excluded); } +// pushes a nested table of the form {plant_index = {growth_id = count}} for a +// map keyed by (plant_index, growth_id) +static void push_growth_map(lua_State* L, const std::map, int>& m) +{ + lua_newtable(L); + for (auto& [key, val] : m) + { + lua_pushinteger(L, key.first); + lua_gettable(L, -2); + if (lua_isnil(L, -1)) + { + lua_pop(L, 1); + lua_newtable(L); + lua_pushinteger(L, key.first); + lua_pushvalue(L, -2); + lua_settable(L, -4); + } + Lua::Push(L, key.second); + lua_pushinteger(L, val); + lua_settable(L, -3); + lua_pop(L, 1); + } +} + static int autofarm_getData(lua_State* L) { lua_newtable(L); @@ -716,8 +899,12 @@ static int autofarm_getData(lua_State* L) Lua::SetField(L, autofarmInstance->getDefault(), ctable, "default_threshold"); Lua::Push(L, autofarmInstance->getThresholds()); lua_setfield(L, ctable, "thresholds"); + push_growth_map(L, autofarmInstance->getGrowthThresholds()); + lua_setfield(L, ctable, "growth_thresholds"); Lua::Push(L, autofarmInstance->getLastCounts()); lua_setfield(L, ctable, "last_counts"); + push_growth_map(L, autofarmInstance->getLastGrowthCounts()); + lua_setfield(L, ctable, "growth_counts"); Lua::Push(L, autofarmInstance->get_excluded_farms()); lua_setfield(L, ctable, "excluded_farms"); return 1; @@ -727,6 +914,8 @@ DFHACK_PLUGIN_LUA_FUNCTIONS { DFHACK_LUA_FUNCTION(autofarm_setDefaultThreshold), DFHACK_LUA_FUNCTION(autofarm_setThreshold), DFHACK_LUA_FUNCTION(autofarm_removeThreshold), + DFHACK_LUA_FUNCTION(autofarm_setGrowthThreshold), + DFHACK_LUA_FUNCTION(autofarm_removeGrowthThreshold), DFHACK_LUA_FUNCTION(autofarm_isFarmExcluded), DFHACK_LUA_FUNCTION(autofarm_setFarmExcluded), DFHACK_LUA_END diff --git a/scripts b/scripts index 2bcaa138db..7e58600967 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 2bcaa138dbd20772e94039d78e8a62850b01e100 +Subproject commit 7e586009670bd55251b443a46b96fb0c7436a47d diff --git a/test/plugins/autofarm.lua b/test/plugins/autofarm.lua index 3936922101..5dfb0a6348 100644 --- a/test/plugins/autofarm.lua +++ b/test/plugins/autofarm.lua @@ -39,6 +39,51 @@ function test.threshold_rejects_bad_plant() or status ~= 0) end +local function find_plant(id) + for _,plant in ipairs(df.global.world.raws.plants.all) do + if plant.id == id then return plant end + end +end + +function test.threshold_growth() + local plugin = require('plugins.autofarm') + -- quarry bushes have a stockpile LEAVES growth in vanilla + local output, status = dfhack.run_command_silent('autofarm', + 'threshold', '30', + 'BUSH_QUARRY:LEAVES') + expect.eq(0, status) + local plant = find_plant('BUSH_QUARRY') + local data = plugin.autofarm_getData() + expect.eq(30, data.growth_thresholds[plant.index].LEAVES) + plugin.autofarm_removeGrowthThreshold(plant.index, 'LEAVES') + plugin.autofarm_removeThreshold(plant.index) +end + +function test.threshold_rejects_bad_growth() + local output, status = dfhack.run_command_silent('autofarm', + 'threshold', '5', + 'BUSH_QUARRY:NOPE') + expect.true_(output:find('Cannot find plant or growth', 1, true) ~= nil + or status ~= 0) + -- growths that are not stockpile-marked are also rejected + output, status = dfhack.run_command_silent('autofarm', + 'threshold', '5', + 'BUCKWHEAT:FLOWERS') + expect.true_(output:find('Cannot find plant or growth', 1, true) ~= nil + or status ~= 0) +end + +function test.growth_threshold_funcs() + local plugin = require('plugins.autofarm') + local plant = find_plant('BUSH_QUARRY') + plugin.autofarm_setGrowthThreshold(plant.index, 'LEAVES', 42) + local data = plugin.autofarm_getData() + expect.eq(42, data.growth_thresholds[plant.index].LEAVES) + plugin.autofarm_removeGrowthThreshold(plant.index, 'LEAVES') + data = plugin.autofarm_getData() + expect.nil_(data.growth_thresholds[plant.index]) +end + function test.farm_exclusion() local plugin = require('plugins.autofarm') expect.false_(plugin.autofarm_isFarmExcluded(-1)) @@ -52,6 +97,8 @@ function test.get_data() local data = require('plugins.autofarm').autofarm_getData() expect.eq('number', type(data.default_threshold)) expect.eq('table', type(data.thresholds)) + expect.eq('table', type(data.growth_thresholds)) expect.eq('table', type(data.last_counts)) + expect.eq('table', type(data.growth_counts)) expect.eq('table', type(data.excluded_farms)) end From 8511cb80387a6e81e1079cc17602b108a747bc0b Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Sat, 31 Oct 2026 17:46:49 -0700 Subject: [PATCH 31/42] Move misplaced entry out of template into 53.16-r3 --- docs/changelog.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog.txt b/docs/changelog.txt index c2470dbe87..4d4ec30a92 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -37,7 +37,6 @@ Template for new versions: ## Misc Improvements ## Documentation -- ``export-world-map``: added more format conversion examples. ## API @@ -64,7 +63,6 @@ Template for new versions: ## Misc Improvements ## Documentation -- ``export-world-map``: added more format conversion examples. ## API @@ -92,6 +90,7 @@ Template for new versions: ## Documentation - ``dfhack.world.isAdventureMode``: doesn't include adventurer arena, advise checking gamemode +- ``export-world-map``: added more format conversion examples. ## API From 29fe02c5685a0a7160621f1eeac819294401931d Mon Sep 17 00:00:00 2001 From: SilasD Date: Sun, 1 Nov 2026 16:04:46 -0800 Subject: [PATCH 32/42] Units.cpp teleport deal with dragged units, pull delete-cage/chained jobs into a predicate, don't break links between wagons and riders, teleport wagon riders with wagons even though they are not on the wagon's tile (although the game seems to tolerate and fix this), RescueWounded jobs set flags4.carrying_somebody not flags1.ridden, remove some redundant namespace specifiers, try to maintain a consistant brace style. --- library/modules/Units.cpp | 108 ++++++++++++++++++++++++++------------ 1 file changed, 73 insertions(+), 35 deletions(-) diff --git a/library/modules/Units.cpp b/library/modules/Units.cpp index 1a41b1b307..3470df9b0c 100644 --- a/library/modules/Units.cpp +++ b/library/modules/Units.cpp @@ -41,6 +41,7 @@ distribution. #include "df/activity_entry.h" #include "df/building_cagest.h" #include "df/building_chainst.h" +#include "df/buildingitemst.h" #include "df/burrow.h" #include "df/caste_raw.h" #include "df/creature_interaction_effect_display_namest.h" @@ -795,10 +796,6 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) return craw && craw->flags.is_set(df::creature_raw_flags::EQUIPMENT_WAGON); }; - auto uid_is_wagon = [&](int32_t unit_id) { - return is_wagon(df::unit::find(unit_id)); - }; - // EQUIPMENT units (e.g. wagons) occupy a 3x3 footprint centered on their // position; all other units occupy just their position tile auto unit_extent = [&](df::unit *unit) { @@ -874,67 +871,108 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) unit->idle_area = target_pos; // Move unit's riders (including babies) to destination - if (unit->flags1.bits.ridden || unit->flags4.bits.carrying_somebody) + if (unit->flags1.bits.ridden || unit->flags4.bits.carrying_somebody) { for (auto rider : world->units.other[units_other_id::ANY_RIDER]) if (rider->relationship_ids[unit_relationship_type::RiderMount] == unit->id) // Wagon riders need special handling -- they are not at the center of the wagon, // and the draft animal riders are not even in the extent of the wagon. - if (is_wagon(unit)) - Units::teleport(rider, (rider->pos - src_pos) + unit->pos); - else - rider->pos = unit->pos; + // Unified this case with the normal case by maintaining the rider's delta-pos from + // the mount, and recursively teleporting instead of just setting the rider's pos. + Units::teleport(rider, (rider->pos - src_pos) + unit->pos); + } - // if the unit itself was a rider, break that link (except do nothing to wagon-riders). - if (unit->flags1.bits.rider) - { - auto mount = df::unit::find(unit->relationship_ids[df::unit_relationship_type::RiderMount]); - if (mount && !is_wagon(mount)) - { + // if the unit itself was a rider and it is not on the same tile as its mount, break that link. + // (except do nothing to wagon-riders). + if (unit->flags1.bits.rider) { + auto mount = df::unit::find(unit->relationship_ids[unit_relationship_type::RiderMount]); + if (mount && unit->pos != mount->pos && !is_wagon(mount)) { unit->flags1.bits.rider = false; - unit->relationship_ids[df::unit_relationship_type::RiderMount] = -1; + unit->relationship_ids[unit_relationship_type::RiderMount] = -1; mount->flags1.bits.ridden = false; mount->flags4.bits.carrying_somebody = false; // rebuild the ridden flags; mothers can carry more than one baby. - for (auto rider : world->units.other[units_other_id::ANY_RIDER]) - if (rider->relationship_ids[unit_relationship_type::RiderMount] == mount->id) - if (rider->profession == df::profession::BABY) + for (auto rider : world->units.other[units_other_id::ANY_RIDER]) { + if (rider->relationship_ids[unit_relationship_type::RiderMount] == mount->id) { + // notes on the recover wounded job: + // the wounded unit has .rider and .move_state true, no relevant refs, + // no job, relationship_ids.RiderMount set, mount_type == 1 Carried, + // health.flags.should_not_move true, a misc_trait of RequestRescueCooldown, + // and an entry in world.units.other.ANY_RIDER. + // the rescuing unit has .carrying_somebody and .move_state true, + // no relevant refs, and a job of type 164 RecoverWounded. + if (rider->profession == df::profession::BABY || (mount->job.current_job + && mount->job.current_job->job_type == job_type::RecoverWounded)) mount->flags4.bits.carrying_somebody = true; else mount->flags1.bits.ridden = true; + } + } } } + // removes jobs related to caging/uncaging/chaining/unchaining the teleported unit. + auto delete_cagee_jobs = [](df::unit* unit, df::building* bld) { + std::vector jobs_to_delete; + for (auto job: bld->jobs) + if (findUnitRef(job->general_refs, df::general_ref_type::UNIT_CAGEE) == unit) + jobs_to_delete.push_back(job); + for (auto job: jobs_to_delete) + Job::removeJob(job); + }; + // If the unit was teleported out of a cage, break those links. - if (auto cage = Units::getContainer(unit)) - { - for (auto job: strict_virtual_cast(Items::getHolderBuilding(cage))->jobs) - if (job->job_type == df::job_type::ReleaseLargeCreature || job->job_type == df::job_type::ReleasePet) - if (auto gref = findRef(job->general_refs, df::general_ref_type::UNIT_CAGEE); - gref && gref->getID() == unit->id) - Job::removeJob(job); - if (auto bld = strict_virtual_cast(Items::getHolderBuilding(cage))) + if (auto cage = getContainer(unit)) { + auto bld = strict_virtual_cast(Items::getHolderBuilding(cage)); + if (bld && bld->contained_items.size() > 0 && bld->contained_items[0]->item == cage) { + delete_cagee_jobs(unit, bld); std::erase(bld->assigned_units, unit->id); + bld->cage_flags.bits.check_occupants = true; + } removeRef(cage->general_refs, df::general_ref_type::CONTAINS_UNIT, unit->id); removeRef(unit->general_refs, df::general_ref_type::CONTAINED_IN_ITEM, cage->id); unit->flags1.bits.caged = false; + // unit->idle_area_type of caged units is set to -1, but that will be corrected by the game soon enough. } // If the unit was teleported out of a chain, break those links. if (auto bld = strict_virtual_cast (findBuildingRef(unit->general_refs, df::general_ref_type::BUILDING_CHAIN))) { - for (auto job: bld->jobs) - if (auto gref = findRef(job->general_refs, df::general_ref_type::UNIT_CAGEE); - (job->job_type == df::job_type::UnchainAnimal || job->job_type == df::job_type::UnchainPet) - && gref && gref->getID() == unit->id) - Job::removeJob(job); + delete_cagee_jobs(unit, bld); if (bld->chained == unit) - bld->chained = NULL; + bld->chained = nullptr; if (bld->assigned == unit) - bld->assigned = NULL; + bld->assigned = nullptr; removeRef(unit->general_refs, df::general_ref_type::BUILDING_CHAIN, bld->id); unit->flags1.bits.chained = false; - } + // unit->idle_area_type of chained units is set to 17 ChainedAnimal, but that will be corrected by the game soon enough. + } + + // removes dragger/draggee links and any dragger job, which is assumed to be related to the dragging. + auto break_dragging_links = [&target_pos](df::unit* dragger, df::unit* draggee) { + if (!dragger || !draggee) + return; + if (dragger->flags1.bits.merchant && draggee->flags1.bits.merchant) { + // this cannot infinite-recurse because at least one of the units is already at target_pos. + if (dragger->pos != target_pos) + Units::teleport(dragger, target_pos); + if (draggee->pos != target_pos) + Units::teleport(draggee, target_pos); + } + if ((isCitizen(dragger, false) || isResident(dragger, false)) && dragger->job.current_job != NULL) + Job::removeJob(dragger->job.current_job); + dragger->relationship_ids[unit_relationship_type::Draggee] = -1; + draggee->relationship_ids[unit_relationship_type::Dragger] = -1; + // draggees have flag1 move_state false, but the game cleans that up during the next tick. + }; + + // If the unit was dragging another unit, break those links. + if (auto draggee = df::unit::find(unit->relationship_ids[unit_relationship_type::Draggee])) + break_dragging_links(unit, draggee); + + // If the unit was being dragged by another unit, break those links. + if (auto dragger = df::unit::find(unit->relationship_ids[unit_relationship_type::Dragger])) + break_dragging_links(dragger, unit); return true; } From c60e8ed0fa69da998c6d82cbcd460562608872b6 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Mon, 2 Nov 2026 10:44:12 -0600 Subject: [PATCH 33/42] ui improvements --- plugins/lua/autofarm.lua | 4 ++-- scripts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/lua/autofarm.lua b/plugins/lua/autofarm.lua index 590e46fb9a..1278aa99dd 100644 --- a/plugins/lua/autofarm.lua +++ b/plugins/lua/autofarm.lua @@ -127,13 +127,13 @@ function FarmCoverOverlay:init() text_pen=COLOR_YELLOW, }, widgets.TextButton{ - frame={b=3, l=2, w=24, h=1}, + frame={b=4, l=2, r=2, h=1}, label='configure autofarm', key='CUSTOM_CTRL_S', on_activate=function() dfhack.run_script('gui/autofarm') end, }, widgets.TextButton{ - frame={b=3, l=30, w=30, h=1}, + frame={b=3, l=2, r=2, h=1}, label='manage this farm manually', key='CUSTOM_CTRL_M', on_activate=function() diff --git a/scripts b/scripts index 7e58600967..ea2ba5d95c 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit 7e586009670bd55251b443a46b96fb0c7436a47d +Subproject commit ea2ba5d95c2376cc798c58c444455573945d374c From 378163cd5a9273670f4ed0d0d98f99371913ea74 Mon Sep 17 00:00:00 2001 From: SilasD Date: Mon, 2 Nov 2026 12:55:25 -0800 Subject: [PATCH 34/42] Units.cpp teleport remove no-longer-riders from ANY_RIDER, riders never are on_ground, fix bug in merchant dragging. --- library/modules/Units.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/library/modules/Units.cpp b/library/modules/Units.cpp index 3470df9b0c..e59274a39c 100644 --- a/library/modules/Units.cpp +++ b/library/modules/Units.cpp @@ -855,7 +855,8 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) } // If there's already somebody standing at the destination, then force the unit to lay down - if (new_occ->bits.unit) + // (except riders never lay down) + if (new_occ->bits.unit && !unit->flags1.bits.rider) unit->flags1.bits.on_ground = true; // Set appropriate occupancy flags at new tile @@ -870,7 +871,7 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) unit->pos = target_pos; unit->idle_area = target_pos; - // Move unit's riders (including babies) to destination + // Move unit's riders (including babies and wounded) to destination if (unit->flags1.bits.ridden || unit->flags4.bits.carrying_somebody) { for (auto rider : world->units.other[units_other_id::ANY_RIDER]) if (rider->relationship_ids[unit_relationship_type::RiderMount] == unit->id) @@ -888,6 +889,8 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) if (mount && unit->pos != mount->pos && !is_wagon(mount)) { unit->flags1.bits.rider = false; unit->relationship_ids[unit_relationship_type::RiderMount] = -1; + // units.other.* are not sorted; do not use erase_from_vector. + std::erase(world->units.other[units_other_id::ANY_RIDER], unit); mount->flags1.bits.ridden = false; mount->flags4.bits.carrying_somebody = false; // rebuild the ridden flags; mothers can carry more than one baby. @@ -958,6 +961,7 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) Units::teleport(dragger, target_pos); if (draggee->pos != target_pos) Units::teleport(draggee, target_pos); + return; } if ((isCitizen(dragger, false) || isResident(dragger, false)) && dragger->job.current_job != NULL) Job::removeJob(dragger->job.current_job); From c0c7bb0486dc1775c925387979cbefcaaee324e1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:11:20 +0000 Subject: [PATCH 35/42] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/python-jsonschema/check-jsonschema: 0.38.2 → 0.39.0](https://github.com/python-jsonschema/check-jsonschema/compare/0.38.2...0.39.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 95a92b054e..7cd0f4c1f1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: args: ['--fix=lf'] - id: trailing-whitespace - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.38.2 + rev: 0.39.0 hooks: - id: check-github-workflows - repo: https://github.com/Lucas-C/pre-commit-hooks From 7c97e51939d3514bf2b8d0cef6cc94d8248a9ace Mon Sep 17 00:00:00 2001 From: SilasD Date: Mon, 2 Nov 2026 14:36:22 -0800 Subject: [PATCH 36/42] Units.cpp teleport don't teleport into walls or impassible building tiles, don't teleport to or from hidden tiles. --- library/modules/Units.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/library/modules/Units.cpp b/library/modules/Units.cpp index e59274a39c..54ae5a3300 100644 --- a/library/modules/Units.cpp +++ b/library/modules/Units.cpp @@ -27,6 +27,7 @@ distribution. #include "Internal.h" #include "MemAccess.h" #include "MiscUtils.h" +#include "Tiletypes.h" #include "Types.h" #include "VersionInfo.h" @@ -789,6 +790,23 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) if (!old_occ || !new_occ) return false; + if (!unit->flags3.bits.ghostly) { + auto new_tt = Maps::getTileType(target_pos); + if (tileShapeBasic(tileShape(*new_tt)) == df::tiletype_shape_basic::Wall) + return false; + + using namespace df::enums::tile_building_occ; + if (auto b = new_occ->bits.building; + b == Obstacle || b == Well || b == Impassable || b == Dynamic) + return false; + } + + // these won't return nullptr because we already got occupancies from these coords. + if (Maps::getTileDesignation(src_pos)->bits.hidden) + return false; + if (Maps::getTileDesignation(target_pos)->bits.hidden) + return false; + auto is_wagon = [](df::unit *unit) { if (!unit) return false; From 8563e1a5fa94dda7978bf6fc7a844cd8910b018c Mon Sep 17 00:00:00 2001 From: SilasD Date: Mon, 2 Nov 2026 15:28:08 -0800 Subject: [PATCH 37/42] Units.cpp teleport update documentation, changelog --- docs/changelog.txt | 6 ++++++ docs/dev/Lua API.rst | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/docs/changelog.txt b/docs/changelog.txt index c2470dbe87..ba6a2a75b3 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -60,8 +60,14 @@ Template for new versions: ## New Features ## Fixes +- ``Units::teleport`` don't teleport units (except ghosts) into walls or impassable parts of buildings; don't teleport into or out of hidden tiles. +- ``Units::teleport`` teleport babies along with their mothers, teleport merchant wagon-riders with their wagons. don't break links between wagons & their riders. +- ``Units::teleport`` when using teleport to separate a rider/carried unit from its mount/carrying unit, clean up the links between the two. +- ``Units::teleport`` when using teleport to separate a dragged unit from a dragger unit, clean up the links between the two. ## Misc Improvements +- ``Units::teleport`` can now remove units from cages & chains, and will cancel any associated uncage/unchain jobs. +- ``Units::teleport`` when teleporting a merchant dragging a pack animal, also teleport the animal. ## Documentation - ``export-world-map``: added more format conversion examples. diff --git a/docs/dev/Lua API.rst b/docs/dev/Lua API.rst index 2f55e17184..823db24607 100644 --- a/docs/dev/Lua API.rst +++ b/docs/dev/Lua API.rst @@ -1808,6 +1808,11 @@ Units module Moves the specified unit and any riders to the target coordinates, setting tile occupancy flags appropriately. Returns true if successful. + This function correctly removes units from cages and chains, and correctly + removes the links between one unit dragging another unit. As a special + case, teleporting a merchant that is dragging a pack animal will also + teleport the pack animal. + * ``dfhack.units.getGeneralRef(unit, type)`` Searches for a ``general_ref`` with the given type. From 20024afcd8101a695a80e22012e075b1a42e5603 Mon Sep 17 00:00:00 2001 From: SilasD Date: Mon, 2 Nov 2026 15:32:09 -0800 Subject: [PATCH 38/42] Units.cpp #include filenames are case-sensitive. --- library/modules/Units.cpp | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/library/modules/Units.cpp b/library/modules/Units.cpp index 54ae5a3300..4c382266c3 100644 --- a/library/modules/Units.cpp +++ b/library/modules/Units.cpp @@ -27,7 +27,7 @@ distribution. #include "Internal.h" #include "MemAccess.h" #include "MiscUtils.h" -#include "Tiletypes.h" +#include "TileTypes.h" #include "Types.h" #include "VersionInfo.h" @@ -907,20 +907,13 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) if (mount && unit->pos != mount->pos && !is_wagon(mount)) { unit->flags1.bits.rider = false; unit->relationship_ids[unit_relationship_type::RiderMount] = -1; - // units.other.* are not sorted; do not use erase_from_vector. + // units.other.* are not sorted; do not use erase_from_vector(). std::erase(world->units.other[units_other_id::ANY_RIDER], unit); mount->flags1.bits.ridden = false; mount->flags4.bits.carrying_somebody = false; // rebuild the ridden flags; mothers can carry more than one baby. for (auto rider : world->units.other[units_other_id::ANY_RIDER]) { if (rider->relationship_ids[unit_relationship_type::RiderMount] == mount->id) { - // notes on the recover wounded job: - // the wounded unit has .rider and .move_state true, no relevant refs, - // no job, relationship_ids.RiderMount set, mount_type == 1 Carried, - // health.flags.should_not_move true, a misc_trait of RequestRescueCooldown, - // and an entry in world.units.other.ANY_RIDER. - // the rescuing unit has .carrying_somebody and .move_state true, - // no relevant refs, and a job of type 164 RecoverWounded. if (rider->profession == df::profession::BABY || (mount->job.current_job && mount->job.current_job->job_type == job_type::RecoverWounded)) mount->flags4.bits.carrying_somebody = true; @@ -931,7 +924,6 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) } } - // removes jobs related to caging/uncaging/chaining/unchaining the teleported unit. auto delete_cagee_jobs = [](df::unit* unit, df::building* bld) { std::vector jobs_to_delete; for (auto job: bld->jobs) @@ -941,7 +933,6 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) Job::removeJob(job); }; - // If the unit was teleported out of a cage, break those links. if (auto cage = getContainer(unit)) { auto bld = strict_virtual_cast(Items::getHolderBuilding(cage)); if (bld && bld->contained_items.size() > 0 && bld->contained_items[0]->item == cage) { @@ -952,10 +943,8 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) removeRef(cage->general_refs, df::general_ref_type::CONTAINS_UNIT, unit->id); removeRef(unit->general_refs, df::general_ref_type::CONTAINED_IN_ITEM, cage->id); unit->flags1.bits.caged = false; - // unit->idle_area_type of caged units is set to -1, but that will be corrected by the game soon enough. } - // If the unit was teleported out of a chain, break those links. if (auto bld = strict_virtual_cast (findBuildingRef(unit->general_refs, df::general_ref_type::BUILDING_CHAIN))) { @@ -966,10 +955,8 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) bld->assigned = nullptr; removeRef(unit->general_refs, df::general_ref_type::BUILDING_CHAIN, bld->id); unit->flags1.bits.chained = false; - // unit->idle_area_type of chained units is set to 17 ChainedAnimal, but that will be corrected by the game soon enough. } - // removes dragger/draggee links and any dragger job, which is assumed to be related to the dragging. auto break_dragging_links = [&target_pos](df::unit* dragger, df::unit* draggee) { if (!dragger || !draggee) return; @@ -985,14 +972,10 @@ bool Units::teleport(df::unit *unit, df::coord target_pos) Job::removeJob(dragger->job.current_job); dragger->relationship_ids[unit_relationship_type::Draggee] = -1; draggee->relationship_ids[unit_relationship_type::Dragger] = -1; - // draggees have flag1 move_state false, but the game cleans that up during the next tick. }; - // If the unit was dragging another unit, break those links. if (auto draggee = df::unit::find(unit->relationship_ids[unit_relationship_type::Draggee])) break_dragging_links(unit, draggee); - - // If the unit was being dragged by another unit, break those links. if (auto dragger = df::unit::find(unit->relationship_ids[unit_relationship_type::Dragger])) break_dragging_links(dragger, unit); From 2f4719fa1113c3b4b87b2f63b748a34b06f365da Mon Sep 17 00:00:00 2001 From: DFHack-Urist via GitHub Actions <63161697+DFHack-Urist@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:53:11 +0000 Subject: [PATCH 39/42] Auto-update submodules library/xml: master scripts: master --- library/xml | 2 +- scripts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/xml b/library/xml index 1196d462d1..2e38688ea0 160000 --- a/library/xml +++ b/library/xml @@ -1 +1 @@ -Subproject commit 1196d462d1c16ba0208be9eb7c58b07e2e955452 +Subproject commit 2e38688ea0e4e20f3a7dd48eabe77458b3a9c19f diff --git a/scripts b/scripts index ea2ba5d95c..d8685b1536 160000 --- a/scripts +++ b/scripts @@ -1 +1 @@ -Subproject commit ea2ba5d95c2376cc798c58c444455573945d374c +Subproject commit d8685b153603aa39c03ad2500f4b0419b427804f From 59d60463964483d3c656d92b5006060be19635ee Mon Sep 17 00:00:00 2001 From: SilasD Date: Tue, 3 Nov 2026 10:50:58 -0800 Subject: [PATCH 40/42] Console-windows: don't advance a line when input reaches last column by printing the line with a method that doesn't advance the cursor position, and by handling the cooked cursor appropriately when it is located at the last column. this also prevents the occasional flicker of the cursor at the beginning of the line whenever the line is updated. note that with this change to output(), it is now identical to blankout(). further work could involve merging the two. alternately, consider moving the creation of a line of spaces into blankout() where it belongs, and changing its function parameters accordingly. --- docs/changelog.txt | 1 + library/Console-windows.cpp | 9 +++------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/changelog.txt b/docs/changelog.txt index c2470dbe87..50098e4638 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -60,6 +60,7 @@ Template for new versions: ## New Features ## Fixes +- Core: Windows console editing will no longer move the cursor to the next line or scroll the screen when the length of the text being edited exceeds the screen width. ## Misc Improvements diff --git a/library/Console-windows.cpp b/library/Console-windows.cpp index bb6b1fc8fb..97178a8e2f 100644 --- a/library/Console-windows.cpp +++ b/library/Console-windows.cpp @@ -223,10 +223,7 @@ namespace DFHack { COORD pos = { (SHORT)x, (SHORT)y }; DWORD count = 0; - CONSOLE_SCREEN_BUFFER_INFO inf = { 0 }; - GetConsoleScreenBufferInfo(console_out, &inf); - SetConsoleCursorPosition(console_out, pos); - WriteConsoleA(console_out, str, len, &count, NULL); + WriteConsoleOutputCharacterA(console_out, str, len, pos, &count); } void prompt_refresh() @@ -237,9 +234,9 @@ namespace DFHack size_t len = raw_buffer.size(); int cooked_cursor = raw_cursor; - if (plen + cooked_cursor > cols) + if (plen + cooked_cursor > (cols-1)) { - int adj = std::min(plen + cooked_cursor - cols, len); + int adj = std::min(plen + cooked_cursor - (cols-1), len); buf += adj; len -= adj; cooked_cursor -= adj; From 1bfe29650ef67f13645c5f2a56ce86556cec79d3 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Thu, 5 Nov 2026 08:39:39 -0600 Subject: [PATCH 41/42] Update CMakeLists.txt --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4247b469f4..8a8223561e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,8 +8,8 @@ set(CMAKE_INSTALL_MESSAGE "LAZY") # set up versioning. set(DF_VERSION "53.16") -set(DFHACK_RELEASE "r3") -set(DFHACK_PRERELEASE FALSE) +set(DFHACK_RELEASE "r4rc1") +set(DFHACK_PRERELEASE TRUE) set(DFHACK_VERSION "${DF_VERSION}-${DFHACK_RELEASE}") set(DFHACK_ABI_VERSION 2) From 43912590d01e32bf49b9eee9b4988f82ca50d569 Mon Sep 17 00:00:00 2001 From: Kelly Kinkade Date: Thu, 5 Nov 2026 10:17:43 -0600 Subject: [PATCH 42/42] fix duplicated/misplaced changelog entries --- docs/changelog.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/changelog.txt b/docs/changelog.txt index 6df3dbfbf8..1473e42e81 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -71,9 +71,7 @@ Template for new versions: - ``Units::teleport`` when using teleport to separate a dragged unit from a dragger unit, clean up the links between the two. ## Misc Improvements -- `autofarm`: farms that span multiple biomes are now handled correctly: crops that can grow in any of the farm's biomes can be assigned to the farm, instead of only using the biome of the farm's center tile - ``eventful``: ``onJobCompleted`` now fires for jobs that are deleted the same tick they complete (e.g. ``HarvestPlants``), detected via the worker's Job unit action timer -- `labormanager`: the modern engine's labor-mismatch probe no longer enables unmanaged labors on idle units or specialists, and no longer re-probes postings that can never be claimed (#6028) - ``RemoteFortressReader``: ensured creature and caste names and caste descriptions in ``GetCreatureRaws``/``GetPartialCreatureRaws`` are transmitted in UTF-8 instead of CP437 - ``Units::teleport`` can now remove units from cages & chains, and will cancel any associated uncage/unchain jobs. - ``Units::teleport`` when teleporting a merchant dragging a pack animal, also teleport the animal.