Merge pull request #22798 from github/nickrolfe/mad-generator-json · github/codeql@781e068 · GitHub
Skip to content

Commit 781e068

Browse files
authored
Merge pull request #22798 from github/nickrolfe/mad-generator-json
MaD: update generator scripts to support writing JSON data extensions
2 parents bc53c78 + 3872102 commit 781e068

5 files changed

Lines changed: 215 additions & 141 deletions

File tree

‎misc/scripts/models-as-data/bulk_generate_mad.py‎

Lines changed: 12 additions & 4 deletions

‎misc/scripts/models-as-data/convert_extensions.py‎

Lines changed: 30 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -8,25 +8,6 @@
88
import tempfile
99

1010

11-
def quote_if_needed(v):
12-
# string columns
13-
if type(v) is str:
14-
return '"' + v + '"'
15-
# bool column
16-
return str(v)
17-
18-
19-
def parseData(data):
20-
rows = [{}, {}]
21-
for row in data:
22-
d = map(quote_if_needed, row)
23-
provenance = row[-1]
24-
targetRows = rows[1] if provenance.endswith("generated") else rows[0]
25-
helpers.insert_update(targetRows, row[0], " - [" + ", ".join(d) + "]\n")
26-
27-
return rows
28-
29-
3011
class Converter:
3112
def __init__(self, language, dbDir):
3213
self.language = language
@@ -38,6 +19,7 @@ def __init__(self, language, dbDir):
3819
)
3920
self.extDir = os.path.join(self.codeQlRoot, f"{self.language}/ql/lib/ext/")
4021
self.dirname = "modelconverter"
22+
# N.B. change this to .model.json to change the output format:
4123
self.modelFileExtension = ".model.yml"
4224
self.workDir = tempfile.mkdtemp()
4325

@@ -63,48 +45,38 @@ def runQuery(self, query):
6345
)
6446
return helpers.readData(self.workDir, resultBqrs)
6547

66-
def asAddsTo(self, rows, predicate):
67-
extensions = [{}, {}]
68-
for i in range(2):
69-
for key in rows[i]:
70-
extensions[i][key] = helpers.addsToTemplate.format(
71-
f"codeql/{self.language}-all", predicate, rows[i][key]
72-
)
73-
74-
return extensions
75-
76-
def getAddsTo(self, query, predicate):
48+
def merge_query_results(self, query, predicate, mergers):
7749
data = self.runQuery(query)
78-
rows = parseData(data)
79-
return self.asAddsTo(rows, predicate)
80-
81-
def makeContent(self):
82-
summaries = self.getAddsTo("ExtractSummaries.ql", helpers.summaryModelPredicate)
83-
sources = self.getAddsTo("ExtractSources.ql", helpers.sourceModelPredicate)
84-
sinks = self.getAddsTo("ExtractSinks.ql", helpers.sinkModelPredicate)
85-
neutrals = self.getAddsTo("ExtractNeutrals.ql", helpers.neutralModelPredicate)
86-
return [
87-
helpers.merge(sources[0], sinks[0], summaries[0], neutrals[0]),
88-
helpers.merge(sources[1], sinks[1], summaries[1], neutrals[1]),
50+
for row in data:
51+
provenance = row[-1]
52+
namespace = row[0]
53+
target_merger = (
54+
mergers[1] if provenance.endswith("generated") else mergers[0]
55+
)
56+
target_merger.add_row(namespace, predicate, row)
57+
58+
def make_extensions(self):
59+
mergers = [
60+
helpers.ExtensionMerger(f"codeql/{self.language}-all"),
61+
helpers.ExtensionMerger(f"codeql/{self.language}-all"),
8962
]
63+
self.merge_query_results(
64+
"ExtractSummaries.ql", helpers.summaryModelPredicate, mergers
65+
)
66+
self.merge_query_results(
67+
"ExtractSources.ql", helpers.sourceModelPredicate, mergers
68+
)
69+
self.merge_query_results("ExtractSinks.ql", helpers.sinkModelPredicate, mergers)
70+
self.merge_query_results(
71+
"ExtractNeutrals.ql", helpers.neutralModelPredicate, mergers
72+
)
73+
return mergers
74+
75+
def run(self):
76+
mergers = self.make_extensions()
9077

91-
def save(self, extensions):
9278
# Create directory if it doesn't exist
9379
os.makedirs(self.extDir, exist_ok=True)
9480

95-
# Create a file for each namespace and save models.
96-
extensionTemplate = """extensions:
97-
{0}"""
98-
for entry in extensions[0]:
99-
with open(self.extDir + "/" + entry + self.modelFileExtension, "w") as f:
100-
f.write(extensionTemplate.format(extensions[0][entry]))
101-
102-
for entry in extensions[1]:
103-
with open(
104-
self.extDir + "/generated/" + entry + self.modelFileExtension, "w"
105-
) as f:
106-
f.write(extensionTemplate.format(extensions[1][entry]))
107-
108-
def run(self):
109-
extensions = self.makeContent()
110-
self.save(extensions)
81+
mergers[0].save(self.extDir, self.modelFileExtension)
82+
mergers[1].save(self.extDir + "/generated", self.modelFileExtension)
Lines changed: 76 additions & 0 deletions

0 commit comments

Comments
 (0)