{{ message }}
Repository navigation
Expand file tree
/
Copy pathcompile-analyzer.ps1
More file actions
321 lines (278 loc) · 12.2 KB
/
Copy pathcompile-analyzer.ps1
File metadata and controls
321 lines (278 loc) · 12.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
<#
.SYNOPSIS
Compile an NLP++ analyzer's KB into kb.dll / kbd.dll for -COMPILED runs.
.DESCRIPTION
Windows counterpart to scripts/compile-analyzer.sh in nlp-engine-linux and
nlp-engine-mac. Three steps:
1. Run nlp.exe -COMPILE on the analyzer + sample input. The engine emits
Cc_code.cpp, Sym*/Con*/Ptr*/St*.cpp, kb_setup.h, and kb_setup.cpp into
<analyzer>/kb/. kb_setup is decorated with __declspec(dllexport) so
MSVC exports it from the resulting DLL (see cc_gen.cpp).
2. Synthesize <analyzer>/.nlp-compile/src/StdAfx.h + CMakeLists.txt and
build the KB shared library in the matching MSVC config.
3. Drop the artifact at <analyzer>/bin/<name>.dll, which is the exact path
cs/libconsh/cg.cpp LoadLibrarys at -COMPILED time. The name depends on
the engine build config:
Debug -> kbd.dll (Debug + UNICODE -> kbdu.dll)
Release -> kb.dll (Release + UNICODE -> kbu.dll)
Engine static libs (prim, kbm, consh, words, lite) come from the local
nlp-engine build tree at <repo>/lib/<Config>/. ICU import libs come from
vcpkg_installed/x64-windows/lib/. Headers come from include/ and cs/include/.
.PARAMETER AnalyzerDir
Path to the analyzer directory (the one that contains spec/ and input/).
.PARAMETER InputFile
A sample input passed to nlp.exe -COMPILE. The compile pass doesn't analyze,
but the engine still requires the arg to be present and resolvable.
.PARAMETER Config
Debug (default) or Release. Picks the matching engine .lib set and names
the output DLL kbd.dll vs kb.dll.
.PARAMETER EngineRoot
Root of the nlp-engine checkout. Defaults to the parent of this script.
.EXAMPLE
scripts\compile-analyzer.ps1 -AnalyzerDir C:\git\analyzers\nlpfix-analyzers\date-time `
-InputFile C:\git\analyzers\nlpfix-analyzers\date-time\input\text.txt
#>
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)] [string] $AnalyzerDir,
[Parameter(Mandatory = $true)] [string] $InputFile,
[ValidateSet('Debug','Release')] [string] $Config = 'Debug',
[string] $EngineRoot
)
$ErrorActionPreference = 'Stop'
function Fail($msg) { Write-Error $msg; exit 1 }
# Resolve EngineRoot relative to this script if not supplied.
if (-not $EngineRoot -or $EngineRoot -eq '') {
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
$EngineRoot = (Resolve-Path (Join-Path $scriptDir '..')).Path
}
$AnalyzerDir = (Resolve-Path $AnalyzerDir).Path
$InputFile = (Resolve-Path $InputFile).Path
$EngineRoot = (Resolve-Path $EngineRoot).Path
# Engine build outputs (multi-config MSVC layout from CMakeLists.txt).
$EngineBin = Join-Path $EngineRoot ("bin\{0}" -f $Config)
$EngineLibDir = Join-Path $EngineRoot ("lib\{0}" -f $Config)
$NlpExe = Join-Path $EngineBin 'nlp.exe'
if (-not (Test-Path -LiteralPath $NlpExe)) {
Fail "nlp.exe not found at $NlpExe. Build the engine first: cmake --build build --config $Config --target nlp"
}
foreach ($name in 'prim','kbm','consh','words','lite') {
if (-not (Test-Path -LiteralPath (Join-Path $EngineLibDir "$name.lib"))) {
Fail "Engine static lib missing: $EngineLibDir\$name.lib. Build target '$name' first."
}
}
# Vcpkg toolchain — drives find_package(ICU) into the correct Debug/Release
# import libs (and matching DLL deployment) for the chosen $Config.
$VcpkgToolchain = Join-Path $EngineRoot 'vcpkg\scripts\buildsystems\vcpkg.cmake'
if (-not (Test-Path -LiteralPath $VcpkgToolchain)) {
Fail "Vcpkg toolchain not found at $VcpkgToolchain. Run: vcpkg\bootstrap-vcpkg.bat then vcpkg\vcpkg.exe install --triplet x64-windows"
}
# Output DLL names the engine LoadLibrarys.
# Compiled KB: cs/libconsh/cg.cpp ~ line 157 — kbd.dll / kb.dll (Debug / Release).
# Compiled rules: lite/nlp.cpp ~ line 1232 — rund.dll / run.dll.
# We're not using UNICODE in the engine build.
$KbDllBase = if ($Config -eq 'Debug') { 'kbd' } else { 'kb' }
$RunDllBase = if ($Config -eq 'Debug') { 'rund' } else { 'run' }
Write-Host ""
Write-Host "==> [1/3] nlp.exe -COMPILE ($AnalyzerDir)"
Write-Host " engine=$NlpExe"
Write-Host ""
# Run -COMPILE. Working dir = engine root so the engine can find data/.
# -WORK passes the same value through to the engine.
& $NlpExe -COMPILE -ANA $AnalyzerDir -WORK $EngineRoot $InputFile
if ($LASTEXITCODE -ne 0) { Fail "nlp.exe -COMPILE exited with code $LASTEXITCODE" }
$KbDir = Join-Path $AnalyzerDir 'kb'
$GeneratedCpp = @(Get-ChildItem -LiteralPath $KbDir -Filter '*.cpp' -ErrorAction SilentlyContinue)
if ($GeneratedCpp.Count -eq 0) {
Fail "No generated .cpp files in $KbDir. Did -COMPILE succeed?"
}
# Build tree under <analyzer>/.nlp-compile/{src,build}.
$BuildRoot = Join-Path $AnalyzerDir '.nlp-compile'
$SrcDir = Join-Path $BuildRoot 'src'
$BuildDir = Join-Path $BuildRoot ("build-{0}" -f $Config)
if (Test-Path -LiteralPath $BuildRoot) { Remove-Item -Recurse -Force -LiteralPath $BuildRoot }
New-Item -ItemType Directory -Force -Path $SrcDir | Out-Null
# Generated sources `#include "StdAfx.h"`. Mirror what the engine's own libs
# use (cs/libconsh/StdAfx.h, plus the kb template): <tchar.h> brings in _TCHAR,
# my_tchar.h brings in the _t_ostream / _T() macros referenced by Cc_code.cpp,
# streamClasses.h is referenced by consh/cg.h.
Set-Content -LiteralPath (Join-Path $SrcDir 'StdAfx.h') -Encoding utf8 -Value @'
#pragma once
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <tchar.h>
#include <my_tchar.h>
#include <streamClasses.h>
'@
# Build CMakeLists.txt. CMake paths use forward slashes on Windows to avoid
# escape headaches inside the here-string.
$AnaCmake = $AnalyzerDir -replace '\\','/'
$EngineRootCmake = $EngineRoot -replace '\\','/'
$EngineLibDirCmake = $EngineLibDir -replace '\\','/'
$SrcDirCmake = $SrcDir -replace '\\','/'
$cmakeLists = @"
cmake_minimum_required(VERSION 3.16)
project(nlp_kb_library LANGUAGES CXX)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
# Build speed. The generated KB is one .cpp per table segment -- parse-en-us
# emits 520 of them, 57 MB, and each expands to ~4.5 MB of identical engine
# headers. Compiling them one at a time took ~15 minutes; batching them into
# unity translation units and letting cl.exe use every core takes ~30 seconds.
#
# /MP matters on its own: `cmake --build --parallel N` with the Visual Studio
# generator becomes msbuild /m:N, which parallelises PROJECTS. There is one
# project here, so without /MP the whole thing compiles serially no matter
# what --parallel says.
#
# Set -DNLP_UNITY_BUILD=OFF to fall back to one TU per file.
option(NLP_UNITY_BUILD "Batch generated sources into unity translation units" ON)
cmake_host_system_information(RESULT NLP_NPROC QUERY NUMBER_OF_LOGICAL_CORES)
if(NOT NLP_NPROC OR NLP_NPROC LESS 1)
set(NLP_NPROC 4)
endif()
# Aim for ~2 batches per core so the pool stays fed, clamped so tiny analyzers
# still spread across cores and huge ones do not build one enormous TU.
function(nlp_set_unity target)
if(NOT NLP_UNITY_BUILD)
return()
endif()
get_target_property(_srcs `${target} SOURCES)
list(LENGTH _srcs _n)
math(EXPR _batch "`${_n} / (`${NLP_NPROC} * 2)")
if(_batch LESS 8)
set(_batch 8)
elseif(_batch GREATER 64)
set(_batch 64)
endif()
set_target_properties(`${target} PROPERTIES
UNITY_BUILD ON
UNITY_BUILD_BATCH_SIZE `${_batch})
message(STATUS "`${target}: `${_n} generated sources, unity batch `${_batch}")
endfunction()
# ICU comes via the vcpkg toolchain (passed on the cmake command line).
# find_package picks the Debug or Release variant to match the active config,
# which keeps the on-disk DLL deps in lockstep with the deployed icu*d?78.dll.
find_package(ICU REQUIRED COMPONENTS i18n uc data io)
# Drop the artifact at <analyzer>/bin/<name>.dll (engine's hardcoded path).
set(KB_OUT_DIR "$AnaCmake/bin")
file(MAKE_DIRECTORY "`${KB_OUT_DIR}")
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "`${KB_OUT_DIR}")
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "`${KB_OUT_DIR}")
foreach(c Debug Release RelWithDebInfo MinSizeRel)
string(TOUPPER `${c} cu)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_`${cu} "`${KB_OUT_DIR}")
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_`${cu} "`${KB_OUT_DIR}")
endforeach()
file(GLOB GENERATED_CPP "$AnaCmake/kb/*.cpp")
if(NOT GENERATED_CPP)
message(FATAL_ERROR "No generated .cpp files in $AnaCmake/kb/")
endif()
add_library(nlp_kb SHARED `${GENERATED_CPP})
set_target_properties(nlp_kb PROPERTIES
OUTPUT_NAME "$KbDllBase"
PREFIX ""
)
target_include_directories(nlp_kb PRIVATE
"$SrcDirCmake"
"$AnaCmake"
"$AnaCmake/kb"
"$EngineRootCmake/include"
"$EngineRootCmake/include/Api"
"$EngineRootCmake/include/Api/lite"
"$EngineRootCmake/cs/include"
)
target_compile_definitions(nlp_kb PRIVATE
_CRT_SECURE_NO_WARNINGS
_CRT_NON_CONFORMING_SWPRINTFS
_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING
NONSTD_
_CONSOLE
WIN32
_WINDOWS
# The engine's own sub-CMakeLists pass MSVC_VERSION (= CMake's MSVC_VERSION
# built-in, e.g. 1941). prim.h:33 uses it to skip the legacy
# #define INFINITY for VS 2015+. Without it the guard sees 0 < 1900 and
# redefines INFINITY, triggering C4005 against ucrt's corecrt_math.h.
MSVC_VERSION=`${MSVC_VERSION}
)
target_compile_options(nlp_kb PRIVATE /FI"StdAfx.h" /Zc:wchar_t /MP)
target_link_directories(nlp_kb PRIVATE
"$EngineLibDirCmake"
)
# Engine static libs. consh + words + kbm depend on prim; lite is top-level.
target_link_libraries(nlp_kb PRIVATE
prim kbm consh words lite
ICU::i18n ICU::uc ICU::data ICU::io
)
nlp_set_unity(nlp_kb)
# --- Compiled analyzer (run.dll / rund.dll) -----------------------------
# Engine looks for <appdir>/bin/rund.dll (Debug) or run.dll (Release) at
# load_compiled time (lite/nlp.cpp ~ line 1232). Empty run/ -> skipped.
file(GLOB RUN_CPP "$AnaCmake/run/*.cpp")
if(RUN_CPP)
add_library(nlp_run SHARED `${RUN_CPP})
set_target_properties(nlp_run PROPERTIES
OUTPUT_NAME "$RunDllBase"
PREFIX ""
)
target_include_directories(nlp_run PRIVATE
"$SrcDirCmake"
"$AnaCmake"
"$AnaCmake/run"
"$AnaCmake/kb"
"$EngineRootCmake/include"
"$EngineRootCmake/include/Api"
"$EngineRootCmake/include/Api/lite"
"$EngineRootCmake/cs/include"
)
target_compile_definitions(nlp_run PRIVATE
_CRT_SECURE_NO_WARNINGS
_CRT_NON_CONFORMING_SWPRINTFS
_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING
NONSTD_
_CONSOLE
WIN32
_WINDOWS
MSVC_VERSION=`${MSVC_VERSION}
)
target_compile_options(nlp_run PRIVATE /FI"StdAfx.h" /Zc:wchar_t /MP)
target_link_directories(nlp_run PRIVATE "$EngineLibDirCmake")
target_link_libraries(nlp_run PRIVATE
prim kbm consh words lite
ICU::i18n ICU::uc ICU::data ICU::io
)
nlp_set_unity(nlp_run)
endif()
"@
Set-Content -LiteralPath (Join-Path $SrcDir 'CMakeLists.txt') -Encoding utf8 -Value $cmakeLists
Write-Host ""
Write-Host "==> [2/3] cmake configure ($BuildDir)"
# VCPKG_INSTALLED_DIR points the vcpkg toolchain at the already-populated
# packages tree under the engine root so find_package(ICU) succeeds without
# requiring a vcpkg.json next to our generated CMakeLists.
$VcpkgInstalled = Join-Path $EngineRoot 'vcpkg_installed'
& cmake -S $SrcDir -B $BuildDir -A x64 `
"-DCMAKE_TOOLCHAIN_FILE=$VcpkgToolchain" `
"-DVCPKG_INSTALLED_DIR=$VcpkgInstalled" `
"-DVCPKG_TARGET_TRIPLET=x64-windows"
if ($LASTEXITCODE -ne 0) { Fail "cmake configure failed (exit $LASTEXITCODE)" }
Write-Host ""
Write-Host "==> [3/3] cmake --build --config $Config"
& cmake --build $BuildDir --config $Config
if ($LASTEXITCODE -ne 0) { Fail "cmake build failed (exit $LASTEXITCODE)" }
$KbOut = Join-Path $AnalyzerDir ("bin\{0}.dll" -f $KbDllBase)
$RunOut = Join-Path $AnalyzerDir ("bin\{0}.dll" -f $RunDllBase)
if (-not (Test-Path -LiteralPath $KbOut)) {
Fail "Expected output $KbOut was not produced."
}
Write-Host ""
Write-Host "Built KB: $KbOut"
if (Test-Path -LiteralPath $RunOut) {
Write-Host "Built run: $RunOut"
} else {
Write-Host "(no run/*.cpp present - analyzer rules will run interpreted)"
}
Write-Host "Run: $NlpExe -COMPILED -ANA $AnalyzerDir -WORK $EngineRoot $InputFile"
Write-Host ""
You can’t perform that action at this time.