1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
|
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0
#
# Copyright (C) Google LLC, 2018
#
# Author: Tom Roeder <tmroeder@google.com>
#
"""A tool for generating compile_commands.json in the Linux kernel."""
import argparse
import json
import logging
import os
import re
import subprocess
_DEFAULT_OUTPUT = 'compile_commands.json'
_DEFAULT_LOG_LEVEL = 'WARNING'
_FILENAME_PATTERN = r'^\..*\.cmd$'
_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$'
_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
# The tools/ directory adopts a different build system, and produces .cmd
# files in a different format. Do not support it.
_EXCLUDE_DIRS = ['.git', 'Documentation', 'include', 'tools']
def parse_arguments():
"""Sets up and parses command-line arguments.
Returns:
log_level: A logging level to filter log output.
directory: The work directory where the objects were built.
ar: Command used for parsing .a archives.
output: Where to write the compile-commands JSON file.
paths: The list of files/directories to handle to find .cmd files.
"""
usage = 'Creates a compile_commands.json database from kernel .cmd files'
parser = argparse.ArgumentParser(description=usage)
directory_help = ('specify the output directory used for the kernel build '
'(defaults to the working directory)')
parser.add_argument('-d', '--directory', type=str, default='.',
help=directory_help)
output_help = ('path to the output command database (defaults to ' +
_DEFAULT_OUTPUT + ')')
parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT,
help=output_help)
log_level_help = ('the level of log messages to produce (defaults to ' +
_DEFAULT_LOG_LEVEL + ')')
parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS,
default=_DEFAULT_LOG_LEVEL, help=log_level_help)
ar_help = 'command used for parsing .a archives'
parser.add_argument('-a', '--ar', type=str, default='llvm-ar', help=ar_help)
paths_help = ('directories to search or files to parse '
'(files should be *.o, *.a, or modules.order). '
'If nothing is specified, the current directory is searched')
parser.add_argument('paths', type=str, nargs='*', help=paths_help)
args = parser.parse_args()
return (args.log_level,
os.path.abspath(args.directory),
args.output,
args.ar,
args.paths if len(args.paths) > 0 else [args.directory])
def cmdfiles_in_dir(directory):
"""Generate the iterator of .cmd files found under the directory.
Walk under the given directory, and yield every .cmd file found.
Args:
directory: The directory to search for .cmd files.
Yields:
The path to a .cmd file.
"""
filename_matcher = re.compile(_FILENAME_PATTERN)
exclude_dirs = [ os.path.join(directory, d) for d in _EXCLUDE_DIRS ]
for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
# Prune unwanted directories.
if dirpath in exclude_dirs:
dirnames[:] = []
continue
for filename in filenames:
if filename_matcher.match(filename):
yield os.path.join(dirpath, filename)
def to_cmdfile(path):
"""Return the path of .cmd file used for the given build artifact
Args:
Path: file path
Returns:
The path to .cmd file
"""
dir, base = os.path.split(path)
return os.path.join(dir, '.' + base + '.cmd')
def cmdfiles_for_o(obj):
"""Generate the iterator of .cmd files associated with the object
Yield the .cmd file used to build the given object
Args:
obj: The object path
Yields:
The path to .cmd file
"""
yield to_cmdfile(obj)
def cmdfiles_for_a(archive, ar):
"""Generate the iterator of .cmd files associated with the archive.
Parse the given archive, and yield every .cmd file used to build it.
Args:
archive: The archive to parse
Yields:
The path to every .cmd file found
"""
for obj in subprocess.check_output([ar, '-t', archive]).decode().split():
yield to_cmdfile(obj)
def cmdfiles_for_modorder(modorder):
"""Generate the iterator of .cmd files associated with the modules.order.
Parse the given modules.order, and yield every .cmd file used to build the
contained modules.
Args:
modorder: The modules.order file to parse
Yields:
The path to every .cmd file found
"""
with open(modorder) as f:
for line in f:
ko = line.rstrip()
base, ext = os.path.splitext(ko)
if ext != '.ko':
sys.exit('{}: module path must end with .ko'.format(ko))
mod = base + '.mod'
# The first line of *.mod lists the objects that compose the module.
with open(mod) as m:
for obj in m.readline().split():
yield to_cmdfile(obj)
def process_line(root_directory, command_prefix, file_path):
"""Extracts information from a .cmd line and creates an entry from it.
Args:
root_directory: The directory that was searched for .cmd files. Usually
used directly in the "directory" entry in compile_commands.json.
command_prefix: The extracted command line, up to the last element.
file_path: The .c file from the end of the extracted command.
Usually relative to root_directory, but sometimes absolute.
Returns:
An entry to append to compile_commands.
Raises:
ValueError: Could not find the extracted file based on file_path and
root_directory or file_directory.
"""
# The .cmd files are intended to be included directly by Make, so they
# escape the pound sign '#', either as '\#' or '$(pound)' (depending on the
# kernel version). The compile_commands.json file is not interepreted
# by Make, so this code replaces the escaped version with '#'.
prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#')
# Use os.path.abspath() to normalize the path resolving '.' and '..' .
abs_path = os.path.abspath(os.path.join(root_directory, file_path))
if not os.path.exists(abs_path):
raise ValueError('File %s not found' % abs_path)
return {
'directory': root_directory,
'file': abs_path,
'command': prefix + file_path,
}
def main():
"""Walks through the directory and finds and parses .cmd files."""
log_level, directory, output, ar, paths = parse_arguments()
level = getattr(logging, log_level)
logging.basicConfig(format='%(levelname)s: %(message)s', level=level)
line_matcher = re.compile(_LINE_PATTERN)
compile_commands = []
for path in paths:
# If 'path' is a directory, handle all .cmd files under it.
# Otherwise, handle .cmd files associated with the file.
# Most of built-in objects are linked via archives (built-in.a or lib.a)
# but some objects are linked to vmlinux directly.
# Modules are listed in modules.order.
if os.path.isdir(path):
cmdfiles = cmdfiles_in_dir(path)
elif path.endswith('.o'):
cmdfiles = cmdfiles_for_o(path)
elif path.endswith('.a'):
cmdfiles = cmdfiles_for_a(path, ar)
elif path.endswith('modules.order'):
cmdfiles = cmdfiles_for_modorder(path)
else:
sys.exit('{}: unknown file type'.format(path))
for cmdfile in cmdfiles:
with open(cmdfile, 'rt') as f:
result = line_matcher.match(f.readline())
if result:
try:
entry = process_line(directory, result.group(1),
result.group(2))
compile_commands.append(entry)
except ValueError as err:
logging.info('Could not add line from %s: %s',
cmdfile, err)
with open(output, 'wt') as f:
json.dump(compile_commands, f, indent=2, sort_keys=True)
if __name__ == '__main__':
main()
|
