BOD 18-01 (web) analysis (#247) · ap713741-svg/https@69f747b · GitHub
Skip to content

Commit 69f747b

Browse files
konklonekonklone
authored
BOD 18-01 (web) analysis (GSA#247)
* making a sub-dir of compliance for m-15-13 analysis work * starting bod-18-01 analysis directory with copies of m-15-13 code * remove scratch space for m-15-13 * handle base domain detection better, header row detection better * update CFO Act agency names as represented in new .gov data * first pass at calculating compliance stats for a single scan event * remove unneeded files from a scan event * ignore libreoffice lock files * proper boolean parsing, ignore unperformed sslyze scans * use parent's preload status regardless of pshtt status for parent * downloading script for pre-bod and post-bod scans * moving data out of version control * finalize download script * removing data from version control, see Releases for this repo * generating first new CSVs for modern reports * some more reports, tweak the column headers some more
1 parent f820a5d commit 69f747b

101 files changed

Lines changed: 1023 additions & 6 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 4 additions & 0 deletions

‎compliance/bod-18-01/compliance.py‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
#!/usr/bin/env python
2+
3+
import utils
4+
import glob
5+
6+
## Todo for DHS:
7+
# * compare data across all dates
8+
9+
## Todo for me:
10+
# * pre-bod path and post-bod path
11+
# * parents-only data going back to 2015?
12+
# * cross-reference with DAP?
13+
14+
# get mapping of domains to agency
15+
base_domains = utils.domains_to_agencies()
16+
17+
# get current and pending preloaded domains
18+
# Downloads live from remote sources. (clear cache/ to re-download)
19+
preloaded = set(utils.fetch_preloaded()) & set(base_domains)
20+
21+
# See download.py for which dates are downloaded for each phase.
22+
23+
# pre-BOD measurements (no 3DES checking, old dir structure)
24+
pre_bod = [x.split("/")[-1] for x in glob.glob("data/pulse/pre-bod/*")]
25+
pre_bod.sort()
26+
27+
# post-BOD measurements (bod_crypto measured, new dir structure)
28+
post_bod = [x.split("/")[-1] for x in glob.glob("data/pulse/post-bod/*")]
29+
post_bod.sort()
30+
31+
32+
# post-bod paths
33+
def pshtts_and_sslyzes_for(date):
34+
pshtts = []
35+
sslyzes = []
36+
37+
if date in post_bod:
38+
pshtts.append("data/pulse/post-bod/%s/parents/results/pshtt.csv" % date)
39+
pshtts.append("data/pulse/post-bod/%s/subdomains/scan/results/pshtt.csv" % date)
40+
sslyzes.append("data/pulse/post-bod/%s/parents/results/sslyze.csv" % date)
41+
sslyzes.append("data/pulse/post-bod/%s/subdomains/scan/results/sslyze.csv" % date)
42+
elif date in pre_bod:
43+
pshtts.append("data/pulse/pre-bod/%s/scan/pshtt.csv" % date)
44+
pshtts.append("data/pulse/pre-bod/%s/subdomains/scan/censys/results/pshtt.csv" % date)
45+
pshtts.append("data/pulse/pre-bod/%s/subdomains/scan/url/results/pshtt.csv" % date)
46+
sslyzes.append("data/pulse/pre-bod/%s/scan/sslyze.csv" % date)
47+
sslyzes.append("data/pulse/pre-bod/%s/subdomains/scan/censys/results/sslyze.csv" % date)
48+
sslyzes.append("data/pulse/pre-bod/%s/subdomains/scan/url/results/sslyze.csv" % date)
49+
50+
return pshtts, sslyzes
51+
52+
def pct(num, denom):
53+
return round((num / denom) * 100)
54+
55+
def compliance_stats(name, date, filter):
56+
pshtts, sslyzes = pshtts_and_sslyzes_for(date)
57+
data = utils.load_pshtt_sslyze(pshtts, sslyzes, base_domains, preloaded, filter=filter)
58+
totals = utils.compliance_totals(data)
59+
60+
count = len(data.keys())
61+
62+
if count == 0:
63+
print("No rows returned, no data to display.")
64+
return
65+
66+
print()
67+
print("=====================================================")
68+
print(" [%s] %s" % (date, name))
69+
print("=====================================================")
70+
print()
71+
print("Total domains: %i" % count)
72+
print()
73+
print("== Direct enforcement ==")
74+
print("Enforces HTTPS: %i (%i%%)" % (totals['enforces'], pct(totals['enforces'], count)))
75+
print("Strong HSTS: %i (%i%%)" % (totals['hsts'], pct(totals['hsts'], count)))
76+
print("RC4 support: %i (%i%%)" % (totals['rc4'], pct(totals['rc4'], count)))
77+
78+
if date in post_bod:
79+
print("3DES support: %i (%i%%)" % (totals['3des'], pct(totals['3des'], count)))
80+
print("Free of Known-weak Crypto: %i (%i%%)" % (totals['bod_crypto'], pct(totals['bod_crypto'], count)))
81+
print("Compliant with BOD 18-01: %i (%i%%)" % (totals['compliant'], pct(totals['compliant'], count)))
82+
83+
elif date in pre_bod:
84+
print("Compliant with M-15-13: %i (%i%%)" % (totals['m1513'], pct(totals['m1513'], count)))
85+
86+
print()
87+
88+
def compliance_csv(dates, when, filter, path):
89+
if when == "pre":
90+
header = [
91+
"Date", "Total Hostnames",
92+
"Enforces HTTPS", "HSTS",
93+
"M-15-13",
94+
"RC4",
95+
"Enforces HTTPS (%)", "HSTS (%)",
96+
"M-15-13 (%)",
97+
"RC4 (%)",
98+
]
99+
elif when == "post":
100+
header = [
101+
"Date", "Total Hostnames",
102+
"Enforces HTTPS", "HSTS",
103+
"M-15-13",
104+
"RC4", "3DES", "Free of SSLv2/SSLv3/RC4/3DES",
105+
"BOD 18-01",
106+
"Enforces HTTPS (%)", "HSTS (%)",
107+
"M-15-13 (%)",
108+
"RC4 (%)", "3DES (%)", "Free of SSLv2/SSLv3/RC4/3DES (%)",
109+
"BOD 18-01 (%)",
110+
]
111+
112+
rows = []
113+
for date in dates:
114+
rows.append(compliance_csv_row(date, filter))
115+
116+
utils.save_csv(header, rows, path)
117+
118+
def compliance_csv_row(date, filter):
119+
print("[%s] Running report for row..." % date)
120+
pshtts, sslyzes = pshtts_and_sslyzes_for(date)
121+
data = utils.load_pshtt_sslyze(pshtts, sslyzes, base_domains, preloaded, filter=filter)
122+
totals = utils.compliance_totals(data)
123+
124+
count = len(data.keys())
125+
126+
if date in pre_bod:
127+
return [
128+
date,
129+
count,
130+
totals['enforces'],
131+
totals['hsts'],
132+
totals['m1513'],
133+
totals['rc4'],
134+
pct(totals['enforces'], count),
135+
pct(totals['hsts'], count),
136+
pct(totals['m1513'], count),
137+
pct(totals['rc4'], count),
138+
]
139+
elif date in post_bod:
140+
return [
141+
date,
142+
count,
143+
totals['enforces'],
144+
totals['hsts'],
145+
totals['m1513'],
146+
totals['rc4'],
147+
totals['3des'],
148+
totals['bod_crypto'],
149+
totals['compliant'],
150+
pct(totals['enforces'], count),
151+
pct(totals['hsts'], count),
152+
pct(totals['m1513'], count),
153+
pct(totals['rc4'], count),
154+
pct(totals['3des'], count),
155+
pct(totals['bod_crypto'], count),
156+
pct(totals['compliant'], count),
157+
]
158+
159+
160+
# # All executive hostnames pre-BOD.
161+
# compliance_csv(pre_bod, "pre",
162+
# utils.executive_only,
163+
# "cache/pre-bod-executive.csv"
164+
# )
165+
# # All executive hostnames post-BOD.
166+
# compliance_csv(post_bod, "post",
167+
# utils.executive_only,
168+
# "cache/post-bod-executive.csv"
169+
# )
170+
171+
172+
# CFO Act
173+
compliance_csv(pre_bod, "pre",
174+
utils.cfo_act_only,
175+
"cache/pre-bod-cfo.csv"
176+
)
177+
compliance_csv(post_bod, "post",
178+
utils.cfo_act_only,
179+
"cache/post-bod-cfo.csv"
180+
)
181+
182+
# Non-CFO Act
183+
compliance_csv(pre_bod, "pre",
184+
utils.executive_non_cfo_act,
185+
"cache/pre-bod-non-cfo.csv"
186+
)
187+
compliance_csv(post_bod, "post",
188+
utils.executive_non_cfo_act,
189+
"cache/post-bod-non-cfo.csv"
190+
)
191+
192+
# # CFO Act (minus DoD)
193+
# compliance_csv(pre_bod, "pre",
194+
# utils.cfo_act_only_sans_dod,
195+
# "cache/pre-bod-cfo-no-dod.csv"
196+
# )
197+
# compliance_csv(post_bod, "post",
198+
# utils.cfo_act_only_sans_dod,
199+
# "cache/post-bod-cfo-no-dod.csv"
200+
# )
201+
202+
# # DoD only
203+
# compliance_csv(pre_bod, "pre",
204+
# utils.for_agencies(["Department of Defense"]),
205+
# "cache/pre-bod-dod.csv"
206+
# )
207+
# compliance_csv(post_bod, "post",
208+
# utils.for_agencies(["Department of Defense"]),
209+
# "cache/post-bod-dod.csv"
210+
# )
211+
212+

‎compliance/bod-18-01/download.py‎

Lines changed: 75 additions & 0 deletions

0 commit comments

Comments
 (0)