#!/usr/bin/env bash # fetch-references.sh — rebuild the external verification environment. # # None of this is redistributed in the archive: UDB and GZDoom are other # people's source, Freedoom is 55 MB, and the converted map corpus is derived. # This script fetches all of it so conformance.js, wadcheck.js and the corpus # sweeps can run. # # ./fetch-references.sh [target-dir] # default ./refs # # Needs network access to github.com, raw.githubusercontent.com and pypi.org. set -euo pipefail DEST="${1:-./refs}" mkdir -p "$DEST" cd "$DEST" echo "fetching into $(pwd)" # ---------------------------------------------------------------- UDB source # Needed for: conformance.js (the .cfg tree), and as the behavioural spec for # Configuration.cs / UniversalParser.cs / UniversalStreamWriter.cs / WAD.cs. # # The guard below asks whether the tree is COMPLETE, not merely whether the # directory exists. `[ ! -d udb ]` alone meant that once refs/ existed it never # changed again — so every file added to the extraction list afterwards was # fetched for a fresh clone and silently missing for everybody else. That is # how Source/Core/Data ended up holding 13 of its 32 files while the glob below # said `Data/*.cs`: the glob was widened after the tree was built. The symptom # is not an error either — citecheck.js reports a citation to a file "nobody # can read", which reads like a bad citation rather than a stale checkout. # # REQUIRED is a manifest of files something in this codebase actually cites. # Add to it when you add a citation to a file not already here. REQUIRED_UDB=( udb/Assets/Common/Configurations/UZDoom_DoomUDMF.cfg udb/Source/Core/IO/Configuration.cs udb/Source/Core/IO/UniversalParser.cs udb/Source/Core/IO/WAD.cs udb/Source/Core/IO/Lump.cs udb/Source/Core/IO/DoomPictureReader.cs udb/Source/Core/IO/DoomFlatReader.cs udb/Source/Core/Controls/NumericTextbox.cs udb/Source/Core/Controls/ButtonsNumericTextbox.cs udb/Source/Core/Controls/PairedFieldsControl.cs udb/Source/Core/Controls/SidedefPartLightControl.cs udb/Source/Core/Controls/CheckboxArrayControl.cs udb/Source/Core/Controls/SectorSlopeControl.cs udb/Source/Core/Controls/SectorSlopeControl.Designer.cs udb/Source/Core/General/General.cs udb/Source/Core/General/MapManager.cs udb/Source/Core/General/MurmurHash2.cs udb/Source/Core/General/StepsList.cs udb/Source/Core/Geometry/Tools.cs udb/Source/Core/Map/MapSet.cs udb/Source/Core/Rendering/Renderer2D.cs udb/Source/Core/Rendering/SurfaceManager.cs udb/Source/Core/Rendering/ViewMode.cs udb/Source/Core/Rendering/Renderer3D.cs udb/Source/Core/VisualModes/VisualMode.cs udb/Source/Core/VisualModes/VisualCamera.cs udb/Source/Core/VisualModes/VisualSector.cs udb/Source/Core/VisualModes/VisualGeometry.cs udb/Source/Core/VisualModes/VisualBlockMap.cs udb/Source/Core/VisualModes/VisualThing.cs udb/Source/Core/GZBuilder/Data/MapInfo.cs # The GZDoom dynamic-light table gzlights.js/gzlightcheck.js/selftest.js # cite 39 times between them. This file was never in the extraction glob # below (only GZBuilder/Data/*.cs was) — it survived on disk only because # nothing had forced a refetch of udb/ since gzlights.js was ported. A # refetch triggered for an unrelated reason (wwwdb-aiu, MapinfoParser.cs) # silently dropped it, which is the exact "file added to the extraction # list afterwards is fetched for a fresh clone and silently missing for # everybody else" failure this script's own header already warns about — # just for a file that was never added to the list at all rather than one # added late. Listing it here is what makes the completeness guard catch # its absence instead of a `rm -rf udb` quietly losing it again. udb/Source/Core/GZBuilder/GZGeneral.cs udb/Source/Core/Rendering/Renderer.cs udb/Source/Core/Map/Sector.cs udb/Builder.sln udb/Assets/Common/Scripting/ZDoom_ACS.cfg udb/Source/Core/Windows/TextureBrowserForm.cs udb/Source/Core/Windows/TextureBrowserForm.Designer.cs udb/Source/Core/Windows/FlatBrowserForm.cs udb/Source/Core/Windows/FlatBrowserForm.Designer.cs udb/Source/Core/Controls/ImageBrowserControl.cs udb/Source/Core/Controls/ImageBrowserControl.Designer.cs udb/Source/Core/Controls/ImageBrowserItem.cs udb/Source/Core/Controls/ImageBrowserCategoryItem.cs udb/Source/Core/Windows/MainForm.Designer.cs udb/Source/Core/Windows/MainForm.cs udb/Source/Core/Actions/Action.cs udb/Source/Core/Actions/ActionManager.cs udb/Source/Core/Actions/SpecialKeys.cs udb/Source/Core/Resources/Actions.cfg # The BUTTON IMAGES, and the two files that say which is which. `mkicons.js` # turns the image NAMES the chrome tables carry into files in `icons/`, and # it needs all three: the loose PNGs, the resource map (a chrome icon names a # RESOURCE, and `Script2` resolves to Script.png — adding ".png" to the name # is wrong for it), and MainForm.resx, which carries one image as base64 # rather than as a reference. udb/Source/Core/Properties/Resources.resx udb/Source/Core/Windows/MainForm.resx # The INFO PANELS, which infoparity.js derives its table from — the designer # for the layout and the companion for which control is a value. udb/Source/Core/Controls/VertexInfoPanel.Designer.cs udb/Source/Core/Controls/VertexInfoPanel.cs udb/Source/Core/Controls/LinedefInfoPanel.Designer.cs udb/Source/Core/Controls/LinedefInfoPanel.cs udb/Source/Core/Controls/ThingInfoPanel.Designer.cs udb/Source/Core/Controls/ThingInfoPanel.cs udb/Source/Core/Controls/SectorInfoPanel.Designer.cs udb/Source/Core/Controls/SectorInfoPanel.cs udb/Source/Core/Controls/StatisticsControl.Designer.cs udb/Source/Core/Controls/StatisticsControl.cs # The DOCKERS panel and the one docker built so far. udb/Source/Core/Controls/Docker.cs udb/Source/Core/Controls/DockersControl.cs udb/Source/Plugins/BuilderModes/Interface/UndoRedoPanel.cs # The four SHAPE draw modes, their OPTION panels — which infoparity.js # derives SHAPE_OPTIONS from, the designer for the layout and the companion # for the ranges — and the geometry they lean on. udb/Source/Plugins/BuilderModes/Interface/DrawRectangleOptionsPanel.Designer.cs udb/Source/Plugins/BuilderModes/Interface/DrawEllipseOptionsPanel.Designer.cs udb/Source/Plugins/BuilderModes/Interface/DrawGridOptionsPanel.Designer.cs udb/Source/Plugins/BuilderModes/Interface/DrawCurveOptionsPanel.Designer.cs udb/Source/Plugins/BuilderModes/ClassicModes/DrawRectangleMode.cs udb/Source/Plugins/BuilderModes/ClassicModes/DrawEllipseMode.cs udb/Source/Plugins/BuilderModes/ClassicModes/DrawGridMode.cs udb/Source/Plugins/BuilderModes/ClassicModes/DrawCurveMode.cs udb/Source/Core/Geometry/CurveTools.cs udb/Source/Core/Geometry/InterpolationTools.cs udb/Source/Core/Resources/Script.png udb/Source/Plugins/BuilderModes/Resources/VerticesMode.png udb/Source/Core/Data/DataManager.cs udb/Source/Core/Data/WADReader.cs udb/Source/Core/Data/TextureImage.cs # The ZDoom TEXTURES text parser (zdtextures.js). ZDTextParser is the # tokeniser; the other three are one lump, one block and one patch. udb/Source/Core/ZDoom/ZDTextParser.cs udb/Source/Core/ZDoom/TexturesParser.cs udb/Source/Core/ZDoom/TextureStructure.cs udb/Source/Core/ZDoom/PatchStructure.cs # The DECORATE actor parser (decorate.js, wwwdb-kdv.1) and the parsed-actor # container it fills (actors.js, wwwdb-kdv / .5). udb/Source/Core/ZDoom/DecorateParser.cs udb/Source/Core/ZDoom/DecorateActorStructure.cs udb/Source/Core/ZDoom/DecorateStateStructure.cs udb/Source/Core/ZDoom/DecorateStateGoto.cs udb/Source/Core/ZDoom/DecorateCategoryInfo.cs udb/Source/Core/ZDoom/ActorStructure.cs udb/Source/Core/ZDoom/StateStructure.cs udb/Source/Core/ZDoom/StateGoto.cs # The ZScript actor parser (zscript.js, wwwdb-kdv.2): a hand-rolled # tokeniser (ZScriptTokenizer, unrelated to ZDTextParser) plus the # class/actor structure it feeds. udb/Source/Core/ZDoom/ZScriptTokenizer.cs udb/Source/Core/ZDoom/ZScriptParser.cs udb/Source/Core/ZDoom/ZScriptActorStructure.cs udb/Source/Core/ZDoom/ZScriptStateStructure.cs udb/Source/Core/ZDoom/ZScriptStateGoto.cs # The MAPINFO parser (wwwdb-aiu): DoomEdNums/SpawnNums, the ONLY route by # which a pure-ZScript actor (no DECORATE stub) gets an editor number. udb/Source/Core/ZDoom/MapinfoParser.cs udb/Source/Core/Data/FlatImage.cs udb/Source/Core/Data/SimpleTextureImage.cs udb/Source/Core/Data/PK3FileImage.cs udb/Source/Core/Data/PK3StructuredReader.cs udb/Source/Core/Data/DirectoryReader.cs udb/Source/Core/Data/PK3Reader.cs udb/Source/Core/Data/FileImage.cs udb/Source/Core/IO/DirectoryFileEntry.cs udb/Source/Core/IO/DirectoryFilesList.cs udb/Source/Core/Data/Playpal.cs udb/Source/Core/Config/ProgramConfiguration.cs udb/Source/Plugins/BuilderModes/General/BuilderPlug.cs udb/Source/Plugins/BuilderModes/ClassicModes/ThingsMode.cs udb/Source/Plugins/BuilderModes/VisualModes/BaseVisualMode.cs udb/Source/Plugins/BuilderModes/Resources/Actions.cfg udb/Source/Plugins/BuilderModes/BuilderModes.csproj udb/Source/Core/Editing/EditModeAttribute.cs udb/Source/Core/Editing/EditModeInfo.cs udb/Source/Core/Editing/EditingManager.cs udb/Source/Core/Config/ThingTypeInfo.cs udb/Source/Core/Config/ConfigurationInfo.cs udb/Source/Core/Config/GameConfiguration.cs udb/Source/Core/Rendering/ColorCollection.cs # The GAME CONFIGURATIONS dialog (Tools > Game Configurations) and everything # it edits: the five-tab form, the resource list control it embeds, the Add / # Edit Resource sub-dialog, and the two data structures underneath — a # DataLocation and the list of them that is written into Builder.cfg under # `configurations..resources`. `Launcher.cs` is here for # `ConvertParameters` alone, which is what the Testing tab's result label # shows. `configparity.js` derives its table from the two Designer files. udb/Source/Core/Windows/ConfigForm.cs udb/Source/Core/Windows/ConfigForm.Designer.cs udb/Source/Core/Windows/ConfigForm.resx udb/Source/Core/Windows/ResourceOptionsForm.cs udb/Source/Core/Windows/ResourceOptionsForm.Designer.cs udb/Source/Core/Windows/ResourceOptionsForm.resx udb/Source/Core/Controls/ResourceListEditor.cs udb/Source/Core/Controls/ResourceListEditor.Designer.cs udb/Source/Core/Controls/ResourceListView.cs udb/Source/Core/Data/DataLocation.cs udb/Source/Core/Data/DataLocationList.cs udb/Source/Core/General/Launcher.cs udb/Source/Core/GZBuilder/Data/EngineInfo.cs udb/Source/Core/Config/NodebuilderInfo.cs udb/Source/Core/Config/DefinedTextureSet.cs udb/Source/Core/Windows/TextureSetForm.cs udb/Source/Core/Windows/TextureSetForm.Designer.cs # The map-side twins of the same resource list, and the PREFERENCES window. # Nothing reads these yet — they are the next two phases' sources, and they # are here so the tree does not need refetching to start one. Note the # basename PreferencesForm.cs now resolves to FOUR files; citecheck.js's # DISAMBIGUATE keeps the existing citations pointed at BuilderModes'. udb/Source/Core/Windows/MapOptionsForm.cs udb/Source/Core/Windows/MapOptionsForm.Designer.cs udb/Source/Core/Windows/OpenMapOptionsForm.cs udb/Source/Core/Windows/OpenMapOptionsForm.Designer.cs udb/Source/Core/Windows/PreferencesForm.cs udb/Source/Core/Windows/PreferencesForm.Designer.cs udb/Source/Core/Windows/PreferencesController.cs ) udb_missing=() for f in "${REQUIRED_UDB[@]}"; do [ -f "$f" ] || udb_missing+=("$f"); done if [ -d udb ] && [ ${#udb_missing[@]} -gt 0 ]; then echo "== UDB tree is INCOMPLETE (${#udb_missing[@]} required file(s) missing) — refetching ==" printf ' missing: %s\n' "${udb_missing[@]}" | head -5 rm -rf udb fi if [ ! -d udb ]; then echo "== Ultimate Doom Builder ==" curl -sL -o udb.tar.gz \ "https://codeload.github.com/UltimateDoomBuilder/UltimateDoomBuilder/tar.gz/refs/heads/master" mkdir -p udb # Config tree plus the source files this project was calibrated against. # # The Windows/, Types/, Config/ and Controls/ entries are for the field model, # and all four are load-bearing rather than nice to have: # Types/*.cs one handler per UniversalType, carrying IsBrowseable / # IsEnumerable / IsLimitedToEnums and UDB's display name. # typeparity.js rebuilds our table from these. # Windows/*EditForm the ONLY place UDB's on-screen wording, control classes # and field order exist. dialogparity.js rebuilds from it. # UniversalFieldInfo.cs the type-coercion rules for a universal field, and # the fact that such a field has a Name and no title. # ArgumentInfo.cs the "Argument N" fallback for an unnamed action arg. # ConfigurationInfo.cs, GameConfiguration.cs which OPTIONAL edit modes a # game configuration turns on, and SupportsMapFeatures. # Both gate what the Mode menu contains, and neither is # in MainForm.Designer.cs. Config/ is small, so the glob # takes the whole directory rather than growing a list # one file at a time and refetching for each. # Leaving any of them out means guessing, and every guess made here so far has # turned out wrong. # Map/*.cs Vertex / Linedef / Sidedef / Sector / MapSet — the # graph topology.js mirrors. Most of the file:line # citations in HANDOFF.md point here, so without these # none of them can be checked. # Geometry/*.cs Tools.cs is DrawLines/MakeSector/FindPotentialSectorAt # itself; Angle2D and Line2D are the primitives whose # exact folding and sign conventions cost two bugs. # Rendering/*.cs Renderer2D, Plotter, Presentation — the phase-4 view. # Data/*.cs graphics decoding AND the resource manager: Playpal, # the Doom patch and flat formats, TEXTUREx composition, # the sprite offsets, plus DataManager/WADReader — which # turn a texture NAME into the bytes for it. # IO/Lump.cs MakeLongName, the hash every texture and flat lookup # is keyed by, and MakeNormalName's NUL-terminated form. # IO/DirectoryFile*.cs the file index a directory or PK3 resource is read # through: the ZDoom 8-character file-title rule, and the # trailing separator that stops "textures/" matching # "textures2/". Reese's own textures are unpacked # directories, so this is not an optional path. # General/MurmurHash2.cs the hash itself, seed and all. Its collision # behaviour is observable, so it is ported exactly # rather than swapped for any other 32-bit hash. # Editing/ClassicMode.cs, ClassicModes/* panning, zooming, mouse # handling, DrawGeometryMode's snap priority, and # BaseClassicMode, which is where the marquee # modifier mapping actually lives — ClassicMode's own # GetMultiSelectionMode returns SELECT unconditionally, # so reading only that file makes ADD/SUBTRACT/INTERSECT # look like dead code. # Builder.sln which plugins SHIP. Source/Plugins holds eighteen # directories and the solution builds fifteen: # ImageDrawingExample, WadAuthorMode and USDF are in the # tree and not in the build. Each of the first two # registers an [EditMode] with a button, so deriving the # Mode menu from the source tree alone puts two modes in # it that no UDB user has ever seen. # Plugins/*/*.cs, Plugins/*/*/*.cs every [EditMode(...)] attribute in the # tree. The MODE MENU is the one menu that is NOT in # MainForm.Designer.cs — the designer gives it two bare # separators and EditingManager fills it at runtime # (EditingManager.cs:271) from the registered modes. So # the menu is derivable, just from a different place: # groups sorted alphabetically, modes within a group # sorted by ButtonOrder. Restricting this to # ClassicModes/ would silently drop Visual Mode and the # six modes other plugins register, and the result would # look complete. # General/BuilderPlug.cs the editing ranges and toggles that are plugin # settings rather than program ones: highlightrange, # stitchrange, mouseselectionthreshold, additiveselect. # ProgramConfiguration.cs the application settings that are NOT in the # .cfg tree (new-sector brightness 192, grid, colours). # Actions/Action.cs, Resources/Actions.cfg the MENU BAR's other half. # MainForm.Designer.cs sets ShortcutKeyDisplayString to # "" on all but one item; ApplyShortcutKeys # (MainForm.cs:2617) fills every one at runtime from # General.Actions[tag].ShortcutKey, and those keys are # declared in Actions.cfg. Without both files the menus # render with no accelerators at all, which looks # finished and is not. # # GNU tar needs --wildcards to glob inside a member list. bsdtar — which is # what macOS ships — globs by default and REJECTS the flag outright, so the # original invocation extracted NOTHING here while `2>/dev/null || true` # swallowed the error and the echo below still printed a count of zero as # though it were progress. That is exactly the "a patch that did not apply # was read as applied" failure section 4 of HANDOFF.md warns about, so this # now detects the flavour AND verifies the result instead of trusting it. TARWILD=(--wildcards) if tar --version 2>&1 | grep -qi bsdtar; then TARWILD=(); fi tar xzf udb.tar.gz -C udb --strip-components=1 ${TARWILD[@]+"${TARWILD[@]}"} \ '*/Assets/Common/Configurations/*' \ '*/Builder.sln' \ '*/Source/Core/IO/Configuration.cs' \ '*/Source/Core/IO/Universal*.cs' \ '*/Source/Core/IO/WAD.cs' \ '*/Source/Core/IO/Lump.cs' \ '*/Source/Core/IO/Doom*Reader.cs' \ '*/Source/Core/IO/DirectoryFile*.cs' \ '*/Source/Core/General/General.cs' \ '*/Source/Core/General/MapManager.cs' \ '*/Source/Core/General/MurmurHash2.cs' \ '*/Source/Core/General/StepsList.cs' \ '*/Source/Core/Map/*.cs' \ '*/Source/Core/Geometry/*.cs' \ '*/Source/Core/Rendering/*.cs' \ '*/Source/Core/VisualModes/*.cs' \ '*/Source/Core/GZBuilder/Data/MapInfo.cs' \ '*/Source/Core/GZBuilder/GZGeneral.cs' \ '*/Source/Core/Data/*.cs' \ '*/Source/Core/ZDoom/*.cs' \ '*/Source/Core/Editing/*.cs' \ '*/Source/Core/Types/*.cs' \ '*/Source/Core/Windows/*EditForm*.cs' \ '*/Source/Core/Windows/*BrowserForm*.cs' \ '*/Assets/Common/Scripting/ZDoom_ACS.cfg' \ '*/Source/Core/Controls/ImageBrowser*.cs' \ '*/Source/Core/Windows/MainForm.cs' \ '*/Source/Core/Windows/MainForm.Designer.cs' \ '*/Source/Core/Windows/ConfigForm*.cs' \ '*/Source/Core/Windows/ConfigForm.resx' \ '*/Source/Core/Windows/ResourceOptionsForm*.cs' \ '*/Source/Core/Windows/ResourceOptionsForm.resx' \ '*/Source/Core/Windows/MapOptionsForm*.cs' \ '*/Source/Core/Windows/OpenMapOptionsForm*.cs' \ '*/Source/Core/Windows/PreferencesForm*.cs' \ '*/Source/Core/Windows/PreferencesController.cs' \ '*/Source/Core/Windows/TextureSetForm*.cs' \ '*/Source/Core/Controls/ResourceList*.cs' \ '*/Source/Core/GZBuilder/Data/EngineInfo.cs' \ '*/Source/Core/General/Launcher.cs' \ '*/Source/Core/Actions/*.cs' \ '*/Source/Core/Resources/Actions.cfg' \ '*/Source/Core/Resources/*.png' \ '*/Source/Core/Properties/Resources.resx' \ '*/Source/Core/Windows/MainForm.resx' \ '*/Source/Plugins/*/Resources/*.png' \ '*/Source/Core/Config/UniversalFieldInfo.cs' \ '*/Source/Core/Config/ArgumentInfo.cs' \ '*/Source/Core/Config/ProgramConfiguration.cs' \ '*/Source/Core/Config/ThingTypeInfo.cs' \ '*/Source/Core/Config/*.cs' \ '*/Source/Core/Controls/*Selector*.cs' \ '*/Source/Core/Controls/ArgumentsControl.cs' \ '*/Source/Core/Controls/FieldsEditorControl*.cs' \ '*/Source/Core/Controls/NumericTextbox.cs' \ '*/Source/Core/Controls/ButtonsNumericTextbox.cs' \ '*/Source/Core/Controls/PairedFieldsControl.cs' \ '*/Source/Core/Controls/SidedefPartLightControl.cs' \ '*/Source/Core/Controls/CheckboxArrayControl.cs' \ '*/Source/Core/Controls/SectorSlopeControl*.cs' \ '*/Source/Core/Controls/*InfoPanel*.cs' \ '*/Source/Core/Controls/StatisticsControl*.cs' \ '*/Source/Core/Controls/Docker*.cs' \ '*/Source/Plugins/BuilderModes/Interface/UndoRedoPanel*.cs' \ '*/Source/Plugins/BuilderModes/Interface/Draw*OptionsPanel*.cs' \ '*/Source/Core/Geometry/CurveTools.cs' \ '*/Source/Core/Geometry/InterpolationTools.cs' \ '*/Source/Plugins/BuilderModes/ClassicModes/*.cs' \ '*/Source/Plugins/*/*.cs' \ '*/Source/Plugins/*/Resources/Actions.cfg' \ '*/Source/Plugins/*/*.csproj' \ '*/Source/Plugins/*/*/*.cs' \ '*/Source/Core/Editing/EditModeAttribute.cs' \ '*/Source/Core/Editing/EditModeInfo.cs' \ '*/Source/Core/Editing/EditingManager.cs' \ '*/Source/Plugins/BuilderModes/General/BuilderPlug.cs' rm -f udb.tar.gz ncfg=$(find udb -name '*.cfg' | wc -l | tr -d ' ') ncs=$(find udb -name '*.cs' | wc -l | tr -d ' ') echo " $ncfg config files, $ncs source files" if [ "$ncfg" -eq 0 ] || [ "$ncs" -eq 0 ]; then echo "ERROR: the UDB tarball extracted nothing. Without it, conformance.js," >&2 echo " typeparity.js and dialogparity.js cannot run and every UDB" >&2 echo " citation in the code is unverifiable. Not continuing." >&2 exit 1 fi # Verify the RESULT rather than trusting the extraction — a nonzero count # says something arrived, not that the right things did. This is the same # check the staleness guard above runs, applied to a fresh tree. after_missing=() for f in "${REQUIRED_UDB[@]}"; do [ -f "$f" ] || after_missing+=("$f"); done if [ ${#after_missing[@]} -gt 0 ]; then echo "ERROR: the UDB tarball extracted, but ${#after_missing[@]} required file(s) are absent:" >&2 printf ' %s\n' "${after_missing[@]}" >&2 echo " Either the extraction list above no longer matches upstream's" >&2 echo " layout, or a file moved. Not continuing." >&2 exit 1 fi fi # ------------------------------------------------------------ GZDoom reader # The engine is the consumer. udmf.cpp holds the coordinate range check that # hard-fails a map load, which no editor-side source expresses. if [ ! -d gzdoom ]; then echo "== GZDoom map loader ==" mkdir -p gzdoom gzdoom/specs for f in src/maploader/udmf.cpp src/maploader/udmf.h src/maploader/maploader.cpp; do curl -sL -o "gzdoom/$(basename "$f")" \ "https://raw.githubusercontent.com/ZDoom/gzdoom/master/$f" done # The specification documents themselves. specparity.js re-derives # defaults.js from these and diffs; three real bugs came out of it the first # time it ran, including two fields whose spec default is CONDITIONAL on # sibling fields and so must not be given a flat default at all. for f in specs/udmf.txt specs/udmf_zdoom.txt; do curl -sL -o "gzdoom/$f" \ "https://raw.githubusercontent.com/ZDoom/gzdoom/master/$f" done echo " $(wc -l < gzdoom/udmf.cpp) lines of udmf.cpp, $(wc -l < gzdoom/specs/udmf.txt) lines of udmf.txt" fi # ------------------------------------------------------------------ Freedoom # Real IWAD container data: 3163 and 3610 lumps, sprite names containing # backslashes, 4-byte lump alignment, marker lumps carrying offsets. if [ ! -f freedoom1.wad ]; then echo "== Freedoom ==" curl -sL -o freedoom.zip \ "https://github.com/freedoom/freedoom/releases/download/v0.13.0/freedoom-0.13.0.zip" python3 - <<'PY' import zipfile z = zipfile.ZipFile('freedoom.zip') for n in z.namelist(): if n.endswith(('freedoom1.wad', 'freedoom2.wad')): open(n.split('/')[-1], 'wb').write(z.read(n)) PY rm -f freedoom.zip echo " $(ls -la freedoom1.wad | awk '{print $5}') bytes freedoom1.wad" fi # ------------------------------------------------- converted UDMF map corpus # Freedoom's maps are binary, so they do not exercise the UDMF parser. omgifol # converts them, and its house style is deliberately unlike UDB's — brace on # the block-name line, no indentation, no spaces around '='. Testing against a # foreign dialect is the point. if [ ! -d udmfmaps ]; then echo "== converting maps with omgifol ==" # A bare `pip` is often not on PATH at all (it was not here), and a # Homebrew python is externally managed, so a plain install refuses. Use a # throwaway venv inside the refs tree instead of --break-system-packages: # it lands in the disposable directory rather than mutating the user's # system python, and it is removed with the rest of refs/. python3 -m venv .venv .venv/bin/python -m pip install -q --upgrade pip .venv/bin/python -m pip install -q omgifol mkdir -p udmfmaps .venv/bin/python - <<'PY' from omg import WAD, UMapEditor n = 0 for wad, tag in [('freedoom1.wad', 'FD1'), ('freedoom2.wad', 'FD2')]: w = WAD(wad) for k in w.maps.keys(): open(f'udmfmaps/{tag}_{k}.TEXTMAP', 'w').write(UMapEditor(w.maps[k]).to_textmap()) n += 1 print(f' {n} maps converted') PY fi # ------------------------------------------------- text encoding (wwwdb-e21) # Runs on EVERY invocation, not just a fresh fetch, because a tree fetched # before this existed still has the problem. # # Two files in UDB's tree are ISO-8859-1 rather than UTF-8 — Vector2D.cs and # CurveLinedefsMode.cs, each for ONE byte: the Å in "Anders Åstrand" in a # credit comment. That byte is not valid UTF-8, so a grep that skips binary # files treats the WHOLE FILE as binary and prints NOTHING — no hits, no # warning, exit 1. `grep` resolves to ugrep with `-I` in the agent shells this # project is driven from, so that is the default experience, not an edge case. # # Not cosmetic. Vector2D.cs is one of the most cited files here, and searching # it is how you check a helper exists before porting a call to it: # `Vector2D.Distance` (Vector2D.cs:228) is called from the `Linedef.Split` # tail. An empty grep over the geometry file is the exact precondition for this # project's one recurring defect — concluding the reference is not there and # deriving a replacement instead. # # Repaired per BYTE, not per file. The first version of this converted any file # that failed a UTF-8 check wholesale with `iconv -f ISO-8859-1`, and that was # wrong twice over: # # * macOS `iconv -f UTF-8 -t UTF-8 file >/dev/null` exits 1 with # "Inappropriate ioctl for device" on files that are perfectly valid, so # the test itself produced false positives. Six valid BOM'd files under # Plugins/ were "normalised" in the trial run and had their BOM # double-encoded. Validity is now decided by a strict UTF-8 decode, which # is what the word means. # * a wholesale re-decode mangles the parts of a mixed file that ARE already # valid UTF-8 (a BOM, a real accented character). Only the invalid bytes # are reinterpreted as Latin-1 now; every valid sequence is left alone. # # ISO-8859-1 -> UTF-8 rewrites one byte as two on one line and can neither add # nor remove a line, so every existing File.cs:line citation still holds. The # line-count assertion below enforces that instead of asserting it in a # comment. echo "== text encoding ==" python3 - <<'ENCPY' import os, io, sys fixed = [] for root, dirs, files in os.walk('.'): for fn in files: if not fn.lower().endswith(('.cs', '.cpp', '.h', '.cfg', '.txt')): continue p = os.path.join(root, fn) data = io.open(p, 'rb').read() try: data.decode('utf-8') continue # already valid: do not touch it except UnicodeDecodeError: pass # Every byte that is not part of a valid UTF-8 sequence comes back as a # lone surrogate U+DCxx. Map just those to the Latin-1 character for # that byte, and leave every valid sequence exactly as it was. s = data.decode('utf-8', errors='surrogateescape') out = ''.join(chr(ord(c) - 0xDC00) if 0xDC80 <= ord(c) <= 0xDCFF else c for c in s) repaired = out.encode('utf-8') if data.count(b'\n') != repaired.count(b'\n'): sys.stderr.write('ERROR: repairing %s changed its line count. Every citation\n' ' into that file would move. Not touching it.\n' % p) sys.exit(1) io.open(p, 'wb').write(repaired) fixed.append(p) for p in fixed: print(' normalised to UTF-8: %s' % p.lstrip('./')) if not fixed: print(' every reference file is already valid UTF-8') ENCPY cat < node typeparity.js $DEST/udb/Source/Core # type table vs UDB handlers node specparity.js $DEST/gzdoom/specs # defaults vs the UDMF spec node dialogparity.js $DEST/udb/Source/Core/Windows # controls vs UDB's forms node wadcheck.js $DEST/freedoom1.wad $DEST/freedoom2.wad for f in $DEST/udmfmaps/*.TEXTMAP; do node harness.js roundtrip "\$f" || echo "FAILED \$f"; done NOT fetched: the reference ZDoom maps. They are Reese's own work, they are the only artifacts that can verify the serializer, and they are ALREADY ON DISK — 22 wads, 21 non-empty — at: ~/Desktop/Work/rhythm doom_orig/firstwad/maps/ Do not ask for them to be re-uploaded, and do not assume the four this script used to name (MAP01/MAP11/MAP21/MAP22) are all there is. Point the corpus sweeps at that directory: node stackcheck.js "\$HOME/Desktop/Work/rhythm doom_orig/firstwad/maps/" node crosscheck.js "\$HOME/Desktop/Work/rhythm doom_orig/firstwad/maps/" EOF