iframe-proxy | Sunbelt Computer Software

177 lines (157 loc) · 6.44 KB
#!/usr/bin/env python3
# Due to various technical limitations, we sometimes have files that need to be
# kept identical in the repository. This script loads a database of such
# files and can perform two functions: check whether they are still identical,
# and overwrite the others with a master copy if needed.
import hashlib
import shutil
import json
path = os.path
file_groups = {}
def add_prefix(prefix, relative):
    result = path.join(prefix, relative)
    if path.commonprefix((path.realpath(result), path.realpath(prefix))) != \
            path.realpath(prefix):
        raise Exception("Path {} is not below {}".format(
            result, prefix))
    return result
def load_if_exists(prefix, json_file_relative):
    json_file_name = path.join(prefix, json_file_relative)
    if path.isfile(json_file_name):
        print("Loading file groups from", json_file_name)
        with open(json_file_name, 'r', encoding='utf-8') as fp:
            raw_groups = json.load(fp)
        prefixed_groups = {
                name: [
                    add_prefix(prefix, relative)
                    for relative in relatives
                for name, relatives in raw_groups.items()
        file_groups.update(prefixed_groups)
# Generates a list of C# test files that should be in sync
def csharp_test_files():
    test_file_re = re.compile('.*(Bad|Good)[0-9]*\\.cs$')
    csharp_doc_files = {
        file:os.path.join(root, file)
            for root, dirs, files in os.walk("csharp/ql/src")
            for file in files
            if test_file_re.match(file)
    return {
        "C# test '" + file + "'" : [os.path.join(root, file), csharp_doc_files[file]]
            for root, dirs, files in os.walk("csharp/ql/test")
            for file in files
            if file in csharp_doc_files
def file_checksum(filename):
    with open(filename, 'rb') as file_handle:
        return hashlib.sha1(file_handle.read()).hexdigest()
def accept_prefix(line1, line2):
    suffix = line2.removeprefix(line1)
    return not suffix or suffix.lstrip().startswith("//")
def equivalent_lines(lines1, lines2):
    if len(lines1) != len(lines2):
        return False
    for line1, line2 in zip(lines1, lines2):
        if not accept_prefix(line1, line2) and not accept_prefix(line2, line1):
            return False
    return True
def check_group(group_name, files, master_file_picker, emit_error, accept_prefix):
    extant_files = [f for f in files if path.isfile(f)]
    if len(extant_files) == 0:
        emit_error(__file__, 0, "No files found from group '" + group_name + "'.")
        emit_error(__file__, 0,
                "Create one of the following files, and then run this script with "
                "the --latest switch to sync it to the other file locations.")
        for filename in files:
            emit_error(__file__, 0, "    " + filename)
        return
    checksums = {file_checksum(f) for f in extant_files}
    same_lengths = len(extant_files) == len(files)
    if len(checksums) == 1 and same_lengths:
        # All files are present and identical.
        return
    # In this case we also consider files indentical, if
    # (1) The group only containts two files.
    # (2) The lines of one file are the same as the lines of another file
    # modulo comments.
    if accept_prefix and same_lengths and len(extant_files) == 2:
        with open(extant_files[0], 'r') as f1:
            file1_lines = [l.strip('\n\r') for l in f1.readlines()]
        with open(extant_files[1], 'r') as f2:
            file2_lines = [l.strip('\n\r') for l in f2.readlines()]
        if equivalent_lines(file1_lines, file2_lines):
            return
    master_file = master_file_picker(extant_files)
    if master_file is None:
        emit_error(__file__, 0,
                "Files from group '"+ group_name +"' not in sync.")
        emit_error(__file__, 0,
                "Run this script with a file-name argument among the "
                "following to overwrite the remaining files with the contents "
                "of that file, or run with the --latest switch to update each "
                "group of files from the most recently modified file in the group.")
        for filename in extant_files:
            emit_error(__file__, 0, "    " + filename)
        print("  Syncing others from", master_file)
        for filename in files:
            if filename == master_file:
                continue
            print("    " + filename)
            if path.isfile(filename):
                os.replace(filename, filename + '~')
            shutil.copy(master_file, filename)
        print("  Backups written with '~' appended to file names")
def chdir_repo_root():
    root_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
    os.chdir(root_path)
def choose_master_file(master_file, files):
    if master_file in files:
        return master_file
        return None
def choose_latest_file(files):
    latest_time = None
    latest_file = None
    for filename in files:
        file_time = os.path.getmtime(filename)
        if (latest_time is None) or (latest_time < file_time):
            latest_time = file_time
            latest_file = filename
    return latest_file
local_error_count = 0
def emit_local_error(path, line, error):
    print('ERROR: ' + path + ':' + str(line) + " - " + error)
    global local_error_count
    local_error_count += 1
# This function is invoked directly by a CI script, which passes a different error-handling
# callback.
def sync_identical_files(emit_error):
    if len(sys.argv) == 1:
        master_file_picker = lambda files: None
    elif len(sys.argv) == 2:
        if sys.argv[1] == "--latest":
            master_file_picker = choose_latest_file
        elif os.path.isfile(sys.argv[1]):
            master_file_picker = lambda files: choose_master_file(sys.argv[1], files)
        else:
            raise Exception("File not found")
        raise Exception("Bad command line or file not found")
    chdir_repo_root()
    load_if_exists('.', 'config/identical-files.json')
    for group_name, files in csharp_test_files().items():
        check_group(group_name, files, master_file_picker, emit_error, True)
    for group_name, files in file_groups.items():
        check_group(group_name, files, master_file_picker, emit_error, False)
def main():
    sync_identical_files(emit_local_error)
    if local_error_count > 0:
        exit(1)
if __name__ == "__main__":
Sunbelt Computer Software

PL/B Language Development and Support

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Sunbelt Computer Software

PL/B Language Development and Support

Uh oh!

FilesExpand file tree

sync-files.py

Latest commit

History

sync-files.py

File metadata and controls
