Skip to content
Navigation Menu
{{ message }}
forked from cirosantilli/linux-kernel-module-cheat
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcommon.py
More file actions
395 lines (368 loc) · 15 KB
/
Copy pathcommon.py
File metadata and controls
395 lines (368 loc) · 15 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
#!/usr/bin/env python3
import argparse
import base64
import copy
import glob
import imp
import os
import re
import shlex
import shutil
import signal
import stat
import subprocess
import sys
this = sys.modules[__name__]
# Default paths.
root_dir = os.path.dirname(os.path.abspath(__file__))
data_dir = os.path.join(root_dir, 'data')
p9_dir = os.path.join(data_dir, '9p')
gem5_non_default_src_root_dir = os.path.join(data_dir, 'gem5')
out_dir = os.path.join(root_dir, 'out')
bench_boot = os.path.join(out_dir, 'bench-boot.txt')
dl_dir = os.path.join(out_dir, 'dl')
submodules_dir = os.path.join(root_dir, 'submodules')
buildroot_src_dir = os.path.join(submodules_dir, 'buildroot')
gem5_default_src_dir = os.path.join(submodules_dir, 'gem5')
linux_src_dir = os.path.join(submodules_dir, 'linux')
qemu_src_dir = os.path.join(submodules_dir, 'qemu')
parsec_benchmark_src_dir = os.path.join(submodules_dir, 'parsec-benchmark')
# Other default variables.
arch_map = {
'a': 'arm',
'A': 'aarch64',
'x': 'x86_64',
}
arches = [arch_map[k] for k in arch_map]
gem5_cpt_prefix = '^cpt\.'
sha = subprocess.check_output(['git', '-C', root_dir, 'log', '-1', '--format=%H']).decode().rstrip()
config_file = os.path.join(data_dir, 'config')
if os.path.exists(config_file):
config = imp.load_source('config', config_file)
configs = {x:getattr(config, x) for x in dir(config) if not x.startswith('__')}
def base64_encode(string):
return base64.b64encode(string.encode()).decode()
def gem_list_checkpoint_dirs():
'''
List checkpoint directory, oldest first.
'''
global this
prefix_re = re.compile(this.gem5_cpt_prefix)
files = list(filter(lambda x: os.path.isdir(os.path.join(this.m5out_dir, x)) and prefix_re.search(x), os.listdir(this.m5out_dir)))
files.sort(key=lambda x: os.path.getmtime(os.path.join(this.m5out_dir, x)))
return files
def get_argparse(default_args=None, argparse_args=None):
'''
Return an argument parser with common arguments set.
:type default_args: Dict[str,str]
:type argparse_args: Dict
'''
global this
if default_args is None:
default_args = {}
if argparse_args is None:
argparse_args = {}
arch_choices = []
for key in this.arch_map:
arch_choices.append(key)
arch_choices.append(this.arch_map[key])
default_build_id = 'default'
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
**argparse_args
)
parser.add_argument(
'-a', '--arch', choices=arch_choices, default='x86_64',
help='CPU architecture. Default: %(default)s'
)
parser.add_argument(
'-g', '--gem5', default=False, action='store_true',
help='Use gem5 instead of QEMU'
)
parser.add_argument(
'-L', '--linux-build-id', default=default_build_id,
help='Linux build ID. Allows you to keep multiple separate Linux builds. Default: %(default)s'
)
parser.add_argument(
'-M', '--gem5-build-id', default=default_build_id,
help='gem5 build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s'
)
parser.add_argument(
'-N', '--gem5-worktree',
help='''\
gem5 git worktree to use for build and Python scripts at runtime. Automatically
create a new git worktree with the given id if one does not exist. If not
given, just use the submodule source.
'''
)
parser.add_argument(
'-n', '--run-id', default='0',
help='''\
ID for run outputs such as gem5's m5out. Allows you to do multiple runs,
and then inspect separate outputs later in different output directories.
Default: %(default)s
'''
)
parser.add_argument(
'--port-offset', type=int,
help='''\
Increase the ports to be used such as for GDB by an offset to run multiple
instances in parallel.
Default: the run ID (-n) if that is an integer, otherwise 0.
'''
)
parser.add_argument(
'-Q', '--qemu-build-id', default=default_build_id,
help='QEMU build ID. Allows you to keep multiple separate QEMU builds. Default: %(default)s'
)
parser.add_argument(
'--buildroot-build-id',
default=default_build_id,
help='Buildroot build ID. Allows you to keep multiple separate gem5 builds. Default: %(default)s'
)
parser.add_argument(
'-t', '--gem5-build-type', default='opt',
help='gem5 build type, most often used for "debug" builds. Default: %(default)s'
)
if hasattr(this, 'configs'):
defaults = this.configs.copy()
else:
defaults = {}
defaults.update(default_args)
# A bit ugly as it actually changes the defaults shown on --help, but we can't do any better
# because it is impossible to check if arguments were given or not...
# - https://stackoverflow.com/questions/30487767/check-if-argparse-optional-argument-is-set-or-not
# - https://stackoverflow.com/questions/3609852/which-is-the-best-way-to-allow-configuration-options-be-overridden-at-the-comman
parser.set_defaults(**defaults)
return parser
def add_build_arguments(parser):
parser.add_argument(
'--clean',
help='Clean the build instead of building.',
action='store_true',
)
def get_elf_entry(elf_file_path):
global this
readelf_header = subprocess.check_output([
this.get_toolchain_tool('readelf'),
'-h',
elf_file_path
])
for line in readelf_header.decode().split('\n'):
split = line.split()
if line.startswith(' Entry point address:'):
addr = line.split()[-1]
break
return int(addr, 0)
def get_stats(stat_re=None, stats_file=None):
global this
if stat_re is None:
stat_re = '^system.cpu[0-9]*.numCycles$'
if stats_file is None:
stats_file = this.stats_file
stat_re = re.compile(stat_re)
ret = []
with open(stats_file, 'r') as statfile:
for line in statfile:
if line[0] != '-':
cols = line.split()
if len(cols) > 1 and stat_re.search(cols[0]):
ret.append(cols[1])
return ret
def get_toolchain_tool(tool):
global this
return glob.glob(os.path.join(this.host_bin_dir, '*-buildroot-*-{}'.format(tool)))[0]
def log_error(msg):
print('error: {}'.format(msg), file=sys.stderr)
def print_cmd(cmd, cmd_file=None, extra_env=None):
'''
Format a command given as a list of strings so that it can
be viewed nicely and executed by bash directly and print it to stdout.
Optionally save the command to cmd_file file, and add extra_env
environment variables to the command generated.
'''
newline_separator = ' \\\n'
out = []
for key in extra_env:
out.extend(['{}={}'.format(shlex.quote(key), shlex.quote(extra_env[key])), newline_separator])
for arg in cmd:
out.extend([shlex.quote(arg), newline_separator])
out = ''.join(out)
print(out)
if cmd_file is not None:
with open(cmd_file, 'w') as f:
f.write('#!/usr/bin/env bash\n')
f.write(out)
st = os.stat(cmd_file)
os.chmod(cmd_file, st.st_mode | stat.S_IXUSR)
def resolve_args(defaults, args, extra_args):
if extra_args is None:
extra_args = {}
argcopy = copy.copy(args)
argcopy.__dict__ = dict(list(defaults.items()) + list(argcopy.__dict__.items()) + list(extra_args.items()))
return argcopy
def rmrf(path):
if os.path.exists(path):
shutil.rmtree(path)
def run_cmd(
cmd,
cmd_file=None,
out_file=None,
show_stdout=True,
show_cmd=True,
extra_env=None,
delete_env=None,
**kwargs
):
'''
Run a command. Write the command to stdout before running it.
Wait until the command finishes execution.
:param cmd: command to run
:type cmd: List[str]
:param cmd_file: if not None, write the command to be run to that file
:type cmd_file: str
:param out_file: if not None, write the stdout and stderr of the command the file
:type out_file: str
:param show_stdout: wether to show stdout and stderr on the terminal or not
:type show_stdout: bool
:param extra_env: extra environment variables to add when running the command
:type extra_env: Dict[str,str]
'''
if out_file is not None:
stdout = subprocess.PIPE
stderr = subprocess.STDOUT
else:
if show_stdout:
stdout = None
stderr = None
else:
stdout = subprocess.DEVNULL
stderr = subprocess.DEVNULL
if extra_env is None:
extra_env = {}
if delete_env is None:
delete_env = []
env = os.environ.copy()
env.update(extra_env)
for key in delete_env:
del env[key]
if show_cmd:
print_cmd(cmd, cmd_file, extra_env=extra_env)
# Otherwise Ctrl + C gives:
# - ugly Python stack trace for gem5 (QEMU takes over terminal and is fine).
# - kills Python, and that then kills GDB: https://stackoverflow.com/questions/19807134/does-python-always-raise-an-exception-if-you-do-ctrlc-when-a-subprocess-is-exec
signal.signal(signal.SIGINT, signal.SIG_IGN)
# https://stackoverflow.com/questions/15535240/python-popen-write-to-stdout-and-log-file-simultaneously/52090802#52090802
with subprocess.Popen(cmd, stdout=stdout, stderr=stderr, env=env, **kwargs) as proc:
if out_file is not None:
with open(out_file, 'bw') as logfile:
while True:
byte = proc.stdout.read(1)
if byte:
if show_stdout:
sys.stdout.buffer.write(byte)
sys.stdout.flush()
logfile.write(byte)
else:
break
signal.signal(signal.SIGINT, signal.SIG_DFL)
return proc.returncode
def setup(parser, **extra_args):
'''
Parse the command line arguments, and setup several variables based on them.
Typically done after getting inputs from the command line arguments.
'''
global this
args = parser.parse_args()
if args.arch in this.arch_map:
args.arch = this.arch_map[args.arch]
if args.arch == 'arm':
this.armv = 7
this.gem5_arch = 'ARM'
elif args.arch == 'aarch64':
this.armv = 8
this.gem5_arch = 'ARM'
elif args.arch == 'x86_64':
this.gem5_arch = 'X86'
this.buildroot_build_dir = os.path.join(this.out_dir, 'buildroot', args.arch, args.buildroot_build_id)
this.buildroot_config_file = os.path.join(this.buildroot_build_dir, '.config')
this.build_dir = os.path.join(this.buildroot_build_dir, 'build')
this.linux_build_dir = os.path.join(this.build_dir, 'linux-custom')
this.linux_variant_dir = '{}.{}'.format(this.linux_build_dir, args.linux_build_id)
this.vmlinux = os.path.join(this.linux_variant_dir, "vmlinux")
this.qemu_build_dir = os.path.join(this.out_dir, 'qemu', args.qemu_build_id)
this.qemu_executable = os.path.join(this.qemu_build_dir, '{}-softmmu'.format(args.arch), 'qemu-system-{}'.format(args.arch))
this.qemu_img_executable = os.path.join(this.qemu_build_dir, 'qemu-img')
this.qemu_guest_build_dir = os.path.join(this.build_dir, 'qemu-custom')
this.host_dir = os.path.join(this.buildroot_build_dir, 'host')
this.host_bin_dir = os.path.join(this.host_dir, 'usr', 'bin')
this.images_dir = os.path.join(this.buildroot_build_dir, 'images')
this.ext2_file = os.path.join(this.images_dir, 'rootfs.ext2')
this.qcow2_file = os.path.join(this.images_dir, 'rootfs.ext2.qcow2')
this.staging_dir = os.path.join(this.buildroot_build_dir, 'staging')
this.target_dir = os.path.join(this.buildroot_build_dir, 'target')
this.run_dir_base = os.path.join(this.out_dir, 'run')
this.gem5_run_dir = os.path.join(this.run_dir_base, 'gem5', args.arch, str(args.run_id))
this.m5out_dir = os.path.join(this.gem5_run_dir, 'm5out')
this.stats_file = os.path.join(this.m5out_dir, 'stats.txt')
this.trace_txt_file = os.path.join(this.m5out_dir, 'trace.txt')
this.gem5_readfile = os.path.join(this.gem5_run_dir, 'readfile')
this.gem5_termout_file = os.path.join(this.gem5_run_dir, 'termout.txt')
this.qemu_run_dir = os.path.join(this.run_dir_base, 'qemu', args.arch, str(args.run_id))
this.qemu_trace_basename = 'trace.bin'
this.qemu_trace_file = os.path.join(this.qemu_run_dir, 'trace.bin')
this.qemu_trace_txt_file = os.path.join(this.qemu_run_dir, 'trace.txt')
this.qemu_termout_file = os.path.join(this.qemu_run_dir, 'termout.txt')
this.qemu_rrfile = os.path.join(this.qemu_run_dir, 'rrfile')
this.gem5_build_dir = os.path.join(this.out_dir, 'gem5', args.gem5_build_id)
this.gem5_m5term = os.path.join(this.gem5_build_dir, 'm5term')
this.gem5_build_build_dir = os.path.join(this.gem5_build_dir, 'build')
this.gem5_executable = os.path.join(this.gem5_build_build_dir, gem5_arch, 'gem5.{}'.format(args.gem5_build_type))
this.gem5_system_dir = os.path.join(this.gem5_build_dir, 'system')
if args.gem5_worktree is not None:
this.gem5_src_dir = os.path.join(this.gem5_non_default_src_root_dir, args.gem5_worktree)
else:
this.gem5_src_dir = this.gem5_default_src_dir
if args.gem5:
this.executable = this.gem5_executable
this.run_dir = this.gem5_run_dir
this.termout_file = this.gem5_termout_file
else:
this.executable = this.qemu_executable
this.run_dir = this.qemu_run_dir
this.termout_file = this.qemu_termout_file
this.gem5_config_dir = os.path.join(this.gem5_src_dir, 'configs')
this.gem5_se_file = os.path.join(this.gem5_config_dir, 'example', 'se.py')
this.gem5_fs_file = os.path.join(this.gem5_config_dir, 'example', 'fs.py')
this.run_cmd_file = os.path.join(this.run_dir, 'run.sh')
if args.arch == 'arm':
this.linux_image = os.path.join('arch', 'arm', 'boot', 'zImage')
elif args.arch == 'aarch64':
this.linux_image = os.path.join('arch', 'arm64', 'boot', 'Image')
elif args.arch == 'x86_64':
this.linux_image = os.path.join('arch', 'x86', 'boot', 'bzImage')
this.linux_image = os.path.join(this.linux_variant_dir, linux_image)
# Ports.
if args.port_offset is None:
try:
args.port_offset = int(args.run_id)
except ValueError:
args.port_offset = 0
if args.gem5:
this.gem5_telnet_port = 3456 + args.port_offset
this.gdb_port = 7000 + args.port_offset
else:
this.qemu_base_port = 45454 + 10 * args.port_offset
this.qemu_monitor_port = this.qemu_base_port + 0
this.qemu_hostfwd_generic_port = this.qemu_base_port + 1
this.qemu_hostfwd_ssh_port = this.qemu_base_port + 2
this.qemu_gdb_port = this.qemu_base_port + 3
this.gdb_port = this.qemu_gdb_port
return args
def mkdir():
global this
os.makedirs(this.build_dir, exist_ok=True)
os.makedirs(this.gem5_build_dir, exist_ok=True)
os.makedirs(this.gem5_run_dir, exist_ok=True)
os.makedirs(this.qemu_run_dir, exist_ok=True)
os.makedirs(this.p9_dir, exist_ok=True)
You can’t perform that action at this time.