WIP: Prototype for pushing DNS · innogames/serveradmin@d026536 · GitHub
Skip to content

Commit d026536

Browse files
brainexebrainexe
committed
WIP: Prototype for pushing DNS
- polishing
1 parent 2ce772b commit d026536

10 files changed

Lines changed: 101 additions & 48 deletions

File tree

‎serveradmin/powerdns/admin.py‎

Lines changed: 1 addition & 1 deletion

‎serveradmin/powerdns/migrations/0010_alter_recordsetting_servertype.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Generated by Django 3.2.23 on 2024-01-09 07:00
2+
3+
from django.db import migrations, models
4+
import django.db.models.deletion
5+
6+
7+
class Migration(migrations.Migration):
8+
9+
dependencies = [
10+
('serverdb', '0017_alter_server_hostname'),
11+
('powerdns', '0009_alter_record_table'),
12+
]
13+
14+
operations = [
15+
migrations.AlterField(
16+
model_name='recordsetting',
17+
name='servertype',
18+
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='serverdb.servertype'),
19+
),
20+
]

‎serveradmin/powerdns/models.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from django.contrib.postgres.fields import ArrayField
12
from django.db import models
23
from django.db.models import Q
34

@@ -14,7 +15,7 @@ class Record(models.Model):
1415
"""PowerDNS Record VIEW representation, so no real table!
1516
"""
1617

17-
object_id = models.IntegerField(primary_key=True)
18+
object_ids = ArrayField(models.IntegerField(), primary_key=True)
1819
name = models.CharField(max_length=255)
1920
type = models.CharField(
2021
max_length=8, choices=[(name.name, name.name) for name in RecordType]
@@ -28,14 +29,14 @@ class Meta:
2829
db_table = "powerdns_records"
2930

3031
def __str__(self):
31-
return f"{self.object_id} {self.name} {self.type} {self.domain}"
32+
return f"{self.object_ids} {self.name} {self.type} {self.domain}"
3233

3334
def get_zone(self):
3435
return get_dns_zone(self.domain)
3536

3637

3738
class RecordSetting(models.Model):
38-
servertype = models.ForeignKey(Servertype, on_delete=models.CASCADE)
39+
servertype = models.ForeignKey(Servertype, on_delete=models.CASCADE, blank=True, null=True)
3940
record_type = models.CharField(
4041
# Support all powerdns record types and an automagic A/AAAA record based on IP family
4142
max_length=8, choices=[("A_AAAA", "A/AAAA")] + [(name.name, name.name) for name in RecordType]

‎serveradmin/powerdns/signals.py‎

Lines changed: 5 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ def create_records(sender, **kwargs):
2323
object_ids = [changed['object_id'] for changed in kwargs['created']]
2424

2525
from serveradmin.powerdns.models import Record
26-
records = Record.objects.filter(object_id__in=object_ids).all()
26+
records = Record.objects.filter(object_ids__contains=object_ids).all()
2727
sync_records(records)
2828

2929

@@ -44,7 +44,7 @@ def delete_records(sender, **kwargs):
4444
logger.info(f"object to delete {kwargs['deleted']}")
4545

4646
# todo fetch deleted entries beforehand to get consistent state to sync?
47-
records = Record.objects.filter(object_id__in=object_ids).all()
47+
records = Record.objects.filter(object_ids__cntains=object_ids).all()
4848
sync_records(records)
4949

5050

@@ -62,28 +62,11 @@ def update_records(sender, **kwargs):
6262
# todo: support changes like this, the following code is not fully working...and not even clean code
6363
# [{'domain': {'action': 'multi', 'add': ['xx3.example.com'], 'remove': ['xx4.example.com']}, 'object_id': 11384}]
6464
# we have to update xx3.example.com + xx4.example.com and not just the new one
65-
from serveradmin.powerdns.models import Record, RecordSetting
66-
67-
# first get all attribute fields which could potentially be related to a DNS record...
68-
domain_attributes = (RecordSetting.objects.exclude(domain__isnull=True)
69-
.distinct('domain').values_list('domain', flat=True))
70-
logger.warning(f"domain_attributes {domain_attributes}")
71-
72-
# then check in the changes and extract all such domain from the before+after changes.
73-
touched_domains = set()
74-
for changes in kwargs['changed']:
75-
logger.warning(f"touched domains {touched_domains}")
76-
77-
for domain_attribute in domain_attributes:
78-
domain_attribute = str(domain_attribute)
79-
if domain_attribute in changes:
80-
touched_domains.update(changes[domain_attribute]['add'])
81-
touched_domains.update(changes[domain_attribute]['remove'])
82-
83-
logger.warning(f"touched domains {touched_domains}")
65+
from serveradmin.powerdns.models import Record
8466

8567
# Is any of the updated objects mapped to a PowerDNS domain?
8668
object_ids = [changed['object_id'] for changed in kwargs['changed']]
87-
records = Record.objects.filter(Q(object_id__in=object_ids)|Q(domain__in=touched_domains)).all()
69+
records = Record.objects.filter(object_ids__contains=object_ids).all()
8870

8971
sync_records(records)
72+
logger.info(f"object to delete {kwargs['changed']} {object_ids}")

‎serveradmin/powerdns/sync/client.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,6 @@ def get_rrsets(self, zone: str, domain_filter: str = '') -> Dict[str, Dict[str,
6060

6161
rrsets = {}
6262
for raw in data:
63-
if raw['type'] == 'SOA':
64-
# we don't care so far!
65-
continue
66-
6763
rrset = RRSet(raw['name'], raw['type'], raw['ttl'])
6864
for record in raw['records']:
6965
rrset.records.add(RecordContent(record['content']))

‎serveradmin/powerdns/sync/objects.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@ def __eq__(self, other):
7272
if self.ttl != other.ttl:
7373
return False
7474
if sorted(self.records) != sorted(other.records):
75+
if self.type == 'SOA':
76+
# todo check serial in the SOA record
77+
return True
7578
return False
7679
return True
7780

‎serveradmin/powerdns/sync/sync.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,21 @@ def sync_records(records: list):
1919
start_time = time.time()
2020

2121
# handle each domain separately and push all touched records to PowerDNS
22-
for zone, records in group_by_zone(records).items():
22+
for zone, zone_records in group_by_zone(records).items():
2323
dns_client.ensure_zone_exists(zone)
2424

2525
# first we have to fetch all relevant existing records from powerdns to check for diff
2626
# in most cases we only need to fetch the records for the domain we are
2727
# currently syncing instead of the whole zone
28-
domain_names = set([record.name for record in records])
28+
domain_names = set([record.name for record in zone_records])
2929
domain_filter = ''
3030
if len(domain_names) == 1:
3131
# todo: find a smarter way to do fetch only relevant records without fetching the whole zone
3232
domain_filter = domain_names.pop()
3333
actual = dns_client.get_rrsets(zone, domain_filter)
3434

3535
# convert our database records to PowerDNS RRSet objects and push via HTTP
36-
expected = db_records_to_pdns_rrsets(zone, records)
36+
expected = db_records_to_pdns_rrsets(zone, zone_records)
3737
diff = get_changed_records(actual, expected)
3838
if not diff:
3939
continue
@@ -137,15 +137,15 @@ def db_records_to_pdns_rrsets(zone: str, records: list) -> List[RRSet]:
137137

138138
# todo: this is a bit hacky, maybe we can configure it somewhere else, as we have to set the NS records for all zones
139139
def get_ns_soa_records(zone: str) -> List[RRSet]:
140-
name_servers = ['ns1.example.com', 'ns2.example.com'] # todo jus some dummy test
140+
name_servers = ['ns1.example.com', 'ns2.example.com'] # todo just some dummy test
141141

142142
ns_rrset = RRSet(zone, 'NS', 3600)
143143
for server in name_servers:
144144
ns_rrset.records.add(RecordContent(ensure_canonical(server)))
145145

146146
soa_rrset = RRSet(zone, 'SOA', 3600)
147147
soa_rrset.records.add(RecordContent(
148-
f"{ensure_canonical(name_servers[0])} hostmaster.{zone}. 1 10800 3600 604800 3600"
148+
f"{ensure_canonical(name_servers[0])} hostmaster.{zone}. 1 86400 7200 2419200 172800"
149149
))
150150

151151
return [ns_rrset, soa_rrset]

‎serveradmin/powerdns/view_sql.py‎

Lines changed: 43 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,11 @@
33

44

55
class ViewSQL:
6-
"""Class to manage the "records" View which represents all configured DNS
7-
records bases on the serveradmin data."""
6+
"""Class to manage the "powerdns_records" View which represents all configured DNS
7+
records bases on the serveradmin data.
8+
The view contains ALL DNS related records in a powerdns-like schema and the "object_ids" which are responsible
9+
for tis record, like vm-server or related domain names
10+
"""
811

912
@classmethod
1013
def update_view_schema(cls):
@@ -20,33 +23,31 @@ def get_record_view_sql(cls):
2023
# - todo: check materialized view with proper indexes
2124
# - todo if view schema is safe, remove it and only REPLACE VIEW
2225
sql = 'DROP VIEW IF EXISTS powerdns_records;'
23-
sql += "CREATE OR REPLACE VIEW powerdns_records (object_id, name, type, content, domain, ttl) AS ("
26+
sql += "CREATE OR REPLACE VIEW powerdns_records (object_ids, name, type, content, domain, ttl) AS ("
2427
sub_queries = []
2528
for record_setting in RecordSetting.objects.all():
29+
object_ids_expression = cls.get_object_ids_expression(record_setting)
2630
name_expression = cls.get_name_expression(record_setting)
2731
type_expression = cls.get_record_type_expression(record_setting)
28-
content_expression = cls.get_content_expression(record_setting)
32+
content_expression = cls.get_content_expression(record_setting, True)
2933
domain_expression = cls.get_domain_expression(record_setting)
3034
attribute_join = cls.get_attribute_join(record_setting)
3135
domain_join = cls.get_domain_join(record_setting)
32-
if record_setting.record_type == 'PTR':
33-
content_expression = 's.hostname'
36+
where_condition = cls.get_where_condition(record_setting)
3437

3538
sub_queries.append(
3639
f"""
3740
SELECT
38-
s.server_id as object_id,
41+
{object_ids_expression} as object_ids,
3942
{name_expression} as name,
4043
{type_expression} as type,
4144
{content_expression} as content,
4245
{domain_expression} as domain,
4346
{record_setting.ttl} as ttl
44-
FROM
45-
server s
47+
FROM server s
4648
{attribute_join}
4749
{domain_join}
48-
WHERE
49-
s.servertype_id = '{record_setting.servertype}'
50+
{where_condition}
5051
"""
5152
)
5253
sql += " UNION ALL ".join(sub_queries)
@@ -55,8 +56,16 @@ def get_record_view_sql(cls):
5556
return sql
5657

5758
@staticmethod
58-
def get_content_expression(record_setting):
59+
def get_content_expression(record_setting, main: bool = False):
60+
if record_setting.record_type == 'PTR' and main:
61+
# todo check if we can simplify PTRs a bit, as we have to reverse a bunch of logic
62+
if record_setting.domain:
63+
return f"domain_name.hostname"
64+
else:
65+
return 's.hostname'
66+
5967
if record_setting.source_value_special:
68+
# the content is in the "server table already"
6069
attribute = Attribute.specials[record_setting.source_value_special]
6170
if attribute.type == 'inet':
6271
# get plain IP address from inet (which also contains the netmask)
@@ -73,6 +82,7 @@ def get_content_expression(record_setting):
7382

7483
@staticmethod
7584
def get_attribute_join(record_setting):
85+
"""if the record content is in an non-special attribute, we need to JOIN the attribute table(s)"""
7686
if record_setting.source_value_special:
7787
# the needed attribute is already in the server table
7888
attribute_join = ""
@@ -103,6 +113,7 @@ def get_record_type_expression(cls, record_setting):
103113

104114
@classmethod
105115
def get_name_expression(cls, record_setting):
116+
"""get the "name" field for the record"""
106117
if record_setting.record_type == 'PTR':
107118
content = cls.get_content_expression(record_setting)
108119
return f"public.ptr({content}::inet)"
@@ -123,6 +134,7 @@ def get_domain_expression(cls, record_setting):
123134

124135
@classmethod
125136
def get_domain_join(cls, record_setting):
137+
"""Get the JOIN for the domain, if needed"""
126138
if record_setting.domain:
127139
return f"""
128140
JOIN server_relation_attribute domain
@@ -131,3 +143,22 @@ def get_domain_join(cls, record_setting):
131143
JOIN server domain_name
132144
ON domain_name.server_id = domain.value"""
133145
return ""
146+
147+
@classmethod
148+
def get_where_condition(cls, record_setting):
149+
"""Add special WHERE conditions to the query, if needed"""
150+
if record_setting.servertype:
151+
return f"WHERE s.servertype_id = '{record_setting.servertype}'"
152+
return ''
153+
154+
@classmethod
155+
def get_object_ids_expression(cls, record_setting):
156+
"""Get the expression to get the object_ids for the record
157+
This are the object_ids of the server and the object_ids of the domain.
158+
So all objects which are kinda touching this record
159+
"""
160+
ids = ['s.server_id']
161+
if record_setting.domain:
162+
ids.append('domain_name.server_id')
163+
164+
return f"ARRAY[{','.join(ids)}]::integer[]"

‎serveradmin/serverdb/migrations/0017_alter_server_hostname.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Generated by Django 3.2.23 on 2024-01-04 08:23
2+
3+
import django.core.validators
4+
from django.db import migrations, models
5+
6+
7+
class Migration(migrations.Migration):
8+
9+
dependencies = [
10+
('serverdb', '0016_optional_servertype_for_relation'),
11+
]
12+
13+
operations = [
14+
migrations.AlterField(
15+
model_name='server',
16+
name='hostname',
17+
field=models.CharField(max_length=254, unique=True, validators=[django.core.validators.RegexValidator('\\A(\\*\\.)?([a-z0-9_]+(\\.|-+))*[a-z0-9]+\\Z', 'Invalid hostname')]),
18+
),
19+
]

‎serveradmin/serverdb/models.py‎

Lines changed: 1 addition & 1 deletion

0 commit comments

Comments
 (0)