FilesExpand file tree

main

/

sync-files.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

177 lines (157 loc) · 6.44 KB

main

/

sync-files.py

Copy path

Top

File metadata and controls

• Code
• Blame

177 lines (157 loc) · 6.44 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

#!/usr/bin/env python3

# Due to various technical limitations, we sometimes have files that need to be

# kept identical in the repository. This script loads a database of such

# files and can perform two functions: check whether they are still identical,

# and overwrite the others with a master copy if needed.

import hashlib

import shutil

import os

import sys

import json

import re

path = os.path

file_groups = {}

def add_prefix(prefix, relative):

result = path.join(prefix, relative)

if path.commonprefix((path.realpath(result), path.realpath(prefix))) != \

path.realpath(prefix):

raise Exception("Path {} is not below {}".format(

result, prefix))

return result

def load_if_exists(prefix, json_file_relative):

json_file_name = path.join(prefix, json_file_relative)

if path.isfile(json_file_name):

print("Loading file groups from", json_file_name)

with open(json_file_name, 'r', encoding='utf-8') as fp:

raw_groups = json.load(fp)

prefixed_groups = {

name: [

add_prefix(prefix, relative)

for relative in relatives

]

for name, relatives in raw_groups.items()

}

file_groups.update(prefixed_groups)

# Generates a list of C# test files that should be in sync

def csharp_test_files():

test_file_re = re.compile('.*(Bad|Good)[0-9]*\\.cs$')

csharp_doc_files = {

file:os.path.join(root, file)

for root, dirs, files in os.walk("csharp/ql/src")

for file in files

if test_file_re.match(file)

}

return {

"C# test '" + file + "'" : [os.path.join(root, file), csharp_doc_files[file]]

for root, dirs, files in os.walk("csharp/ql/test")

for file in files

if file in csharp_doc_files

}

def file_checksum(filename):

with open(filename, 'rb') as file_handle:

return hashlib.sha1(file_handle.read()).hexdigest()

def accept_prefix(line1, line2):

suffix = line2.removeprefix(line1)

return not suffix or suffix.lstrip().startswith("//")

def equivalent_lines(lines1, lines2):

if len(lines1) != len(lines2):

return False

for line1, line2 in zip(lines1, lines2):

if not accept_prefix(line1, line2) and not accept_prefix(line2, line1):

return False

return True

def check_group(group_name, files, master_file_picker, emit_error, accept_prefix):

extant_files = [f for f in files if path.isfile(f)]

if len(extant_files) == 0:

emit_error(__file__, 0, "No files found from group '" + group_name + "'.")

emit_error(__file__, 0,

"Create one of the following files, and then run this script with "

"the --latest switch to sync it to the other file locations.")

for filename in files:

emit_error(__file__, 0, " " + filename)

return

checksums = {file_checksum(f) for f in extant_files}

same_lengths = len(extant_files) == len(files)

if len(checksums) == 1 and same_lengths:

# All files are present and identical.

return

# In this case we also consider files indentical, if

# (1) The group only containts two files.

# (2) The lines of one file are the same as the lines of another file

# modulo comments.

if accept_prefix and same_lengths and len(extant_files) == 2:

with open(extant_files[0], 'r') as f1:

file1_lines = [l.strip('\n\r') for l in f1.readlines()]

with open(extant_files[1], 'r') as f2:

file2_lines = [l.strip('\n\r') for l in f2.readlines()]

if equivalent_lines(file1_lines, file2_lines):

return

master_file = master_file_picker(extant_files)

if master_file is None:

emit_error(__file__, 0,

"Files from group '"+ group_name +"' not in sync.")

emit_error(__file__, 0,

"Run this script with a file-name argument among the "

"following to overwrite the remaining files with the contents "

"of that file, or run with the --latest switch to update each "

"group of files from the most recently modified file in the group.")

for filename in extant_files:

emit_error(__file__, 0, " " + filename)

else:

print(" Syncing others from", master_file)

for filename in files:

if filename == master_file:

continue

print(" " + filename)

if path.isfile(filename):

os.replace(filename, filename + '~')

shutil.copy(master_file, filename)

print(" Backups written with '~' appended to file names")

def chdir_repo_root():

root_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')

os.chdir(root_path)

def choose_master_file(master_file, files):

if master_file in files:

return master_file

else:

return None

def choose_latest_file(files):

latest_time = None

latest_file = None

for filename in files:

file_time = os.path.getmtime(filename)

if (latest_time is None) or (latest_time < file_time):

latest_time = file_time

latest_file = filename

return latest_file

local_error_count = 0

def emit_local_error(path, line, error):

print('ERROR: ' + path + ':' + str(line) + " - " + error)

global local_error_count

local_error_count += 1

# This function is invoked directly by a CI script, which passes a different error-handling

# callback.

def sync_identical_files(emit_error):

if len(sys.argv) == 1:

master_file_picker = lambda files: None

elif len(sys.argv) == 2:

if sys.argv[1] == "--latest":

master_file_picker = choose_latest_file

elif os.path.isfile(sys.argv[1]):

master_file_picker = lambda files: choose_master_file(sys.argv[1], files)

else:

raise Exception("File not found")

else:

raise Exception("Bad command line or file not found")

chdir_repo_root()

load_if_exists('.', 'config/identical-files.json')

for group_name, files in csharp_test_files().items():

check_group(group_name, files, master_file_picker, emit_error, True)

for group_name, files in file_groups.items():

check_group(group_name, files, master_file_picker, emit_error, False)

def main():

sync_identical_files(emit_local_error)

if local_error_count > 0:

exit(1)

if __name__ == "__main__":

main()