[pull] master from Comfy-Org:master by pull[bot] · Pull Request #1149 · code/app-python-comfyui · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
56 changes: 53 additions & 3 deletions app/assets/helpers.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
"""Small conversions the asset system needs in more than one layer: canonical
stored-hash strings, UTC timestamps, tag normalization, and a SQL predicate for
path containment. That predicate is deliberately case-sensitive and
component-bounded, because callers use it to choose rows for hard deletion.
stored-hash strings, UTC timestamps, tag normalization, and path containment
checks. The SQL predicate is deliberately case-sensitive and component-bounded,
because callers use it to choose rows for hard deletion; the Python matcher
follows ``Path.is_relative_to`` instead, including its platform case rules.
"""

import functools
import os
from collections.abc import Callable, Iterable
from datetime import datetime, timezone

import sqlalchemy as sa
Expand Down Expand Up @@ -42,6 +45,53 @@ def sql_path_under_prefix(
)


def path_prefix_matcher(prefixes: Iterable[str]) -> Callable[[str], bool]:
"""Return ``path -> Path(path).is_relative_to(<any prefix>)``, with the prefixes
normalized once.

The startup prune tests every catalogued row against every owned prefix, and
``Path.is_relative_to`` walks the path's parents on each call, so a pathlib
check there costs rows x prefixes x depth. A normcase'd, separator-bounded
string prefix keeps its component bounds and platform case rules.
"""
# abspath keeps exactly two leading separators, and pathlib treats that "//" as an
# anchor of its own: "//server/f" is not under "/". Such paths are only matched
# against prefixes with the same anchor.
double = os.sep * 2
exact: dict[bool, set[str]] = {False: set(), True: set()}
stems: dict[bool, list[str]] = {False: [], True: []}
for prefix in prefixes:
base = os.path.normcase(os.path.abspath(prefix))
is_double = base.startswith(double)
exact[is_double].add(base)
stems[is_double].append(base if base.endswith(os.sep) else base + os.sep)
stem_tuples = {key: tuple(value) for key, value in stems.items()}

def matches(path: str) -> bool:
candidate = os.path.normcase(os.path.abspath(path))
is_double = candidate.startswith(double)
return candidate in exact[is_double] or candidate.startswith(stem_tuples[is_double])

return matches


@functools.lru_cache(maxsize=None)
def cached_prefix_matcher(prefixes: tuple[str, ...]) -> Callable[[str], bool]:
"""path_prefix_matcher, built once per distinct prefix tuple, for callers that check
every scanned file against the same folders.

Keyed on the raw prefixes: normalizing them in the key would bring back the per-call
cost this avoids. A folder-config change is just a new key.

Precondition: the prefixes are absolute. That is not checked. main.py and
extra_model_paths make their folders absolute, but folder_paths' setters and
add_model_folder_path store whatever they are given, so a custom node can register a
relative one. A relative prefix is resolved against the working directory once, on
first use, and that resolution is then frozen for as long as the key is unchanged.
"""
return path_prefix_matcher(prefixes)


def escape_sql_like_string(s: str, escape: str = "!") -> tuple[str, str]:
"""Escapes %, _ and the escape char in a LIKE prefix.

Expand Down
10 changes: 3 additions & 7 deletions app/assets/scanner.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,12 @@
create_record,
)
from app.assets.database.models import Asset, AssetContent
from app.assets.helpers import sql_path_under_prefix, to_stored_hash
from app.assets.helpers import path_prefix_matcher, sql_path_under_prefix, to_stored_hash
from app.assets.lifecycle import get_excluded_scan_roots
from app.assets.scanner_changes import (
clear_pending_verifications,
detect_content_change,
drain_pending_verifications,
is_path_under_prefixes,
live_contents_under_prefixes,
pending_recovery_count,
recover_missing_content,
Expand Down Expand Up @@ -240,10 +239,6 @@ def _sync_prefixes_in_write_txn(
return survivors


def _is_under_prefixes(path: str, prefixes: list[str]) -> bool:
return is_path_under_prefixes(path, prefixes)


def sync_root_safely(
root: RootType, progress: _ScanProgress | None = None
) -> set[str]:
Expand Down Expand Up @@ -304,7 +299,8 @@ def mark_contents_missing_outside_prefixes(
contents = session.scalars(
sa.select(AssetContent).where(AssetContent.is_missing.is_(False))
)
missing = [content for content in contents if not _is_under_prefixes(content.path, prefixes)]
is_owned = path_prefix_matcher(prefixes)
missing = [content for content in contents if not is_owned(content.path)]
for content in missing:
mark_content_missing(session, content.id)
return len(missing)
Expand Down
6 changes: 2 additions & 4 deletions app/assets/scanner_changes.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
from __future__ import annotations

import os
from pathlib import Path
from typing import Literal

import sqlalchemy as sa
Expand All @@ -21,7 +20,7 @@
mark_content_missing,
unset_content_missing,
)
from app.assets.helpers import sql_path_under_prefix, to_stored_hash
from app.assets.helpers import path_prefix_matcher, sql_path_under_prefix, to_stored_hash
from app.assets.services.path_utils import compute_loader_path, get_name_and_tags_from_asset_path
from app.assets.services.snapshot_hash import snapshot_hash

Expand Down Expand Up @@ -108,8 +107,7 @@ def recover_missing_content(


def is_path_under_prefixes(path: str, prefixes: list[str]) -> bool:
candidate = Path(os.path.abspath(path))
return any(candidate.is_relative_to(os.path.abspath(prefix)) for prefix in prefixes)
return path_prefix_matcher(prefixes)(path)


def split_content(session: Session, content: AssetContent, stat_result: os.stat_result, hash_value: str | None) -> AssetContent:
Expand Down
20 changes: 9 additions & 11 deletions app/assets/services/path_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from typing import Literal

import folder_paths
from app.assets.helpers import cached_prefix_matcher


_NON_MODEL_FOLDER_NAMES = frozenset({"configs", "custom_nodes"})
Expand Down Expand Up @@ -250,7 +251,6 @@ def get_backend_system_tags_from_path(path: str) -> list[str]:
category still get the ``models`` tag.
"""
fp_abs = os.path.abspath(path)
fp_path = Path(fp_abs)
tags: list[str] = []

def _add(tag: str) -> None:
Expand All @@ -262,21 +262,19 @@ def _add(tag: str) -> None:
("output", folder_paths.get_output_directory()),
("temp", folder_paths.get_temp_directory()),
):
if fp_path.is_relative_to(os.path.abspath(base)):
if cached_prefix_matcher((base,))(fp_abs):
_add(role)

ext = os.path.splitext(fp_abs)[1].lower()
model_types: list[str] = []
under_models_base = False
for folder_name, bases, extensions in get_comfy_models_folders():
for base in bases:
if fp_path.is_relative_to(os.path.abspath(base)):
under_models_base = True
# Empty set accepts any extension, matching
# folder_paths.filter_files_extensions semantics.
if not extensions or ext in extensions:
model_types.append(folder_name)
break
if cached_prefix_matcher(tuple(bases))(fp_abs):
under_models_base = True
# Empty set accepts any extension, matching
# folder_paths.filter_files_extensions semantics.
if not extensions or ext in extensions:
model_types.append(folder_name)

if under_models_base:
_add("models")
Expand Down Expand Up @@ -306,7 +304,7 @@ def get_known_input_subfolder_tags_from_path(path: str) -> list[str]:
"""
fp_abs = os.path.abspath(path)
input_base = os.path.abspath(folder_paths.get_input_directory())
if not Path(fp_abs).is_relative_to(input_base):
if not cached_prefix_matcher((input_base,))(fp_abs):
return []

rel = os.path.relpath(fp_abs, input_base)
Expand Down
72 changes: 69 additions & 3 deletions comfy/audio_encoders/sheetsage2_abc.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import math
import re
from bisect import bisect_left
from collections import Counter
from dataclasses import dataclass, replace
from fractions import Fraction
Expand Down Expand Up @@ -52,10 +53,12 @@ def events_to_abc(events, duration, melody_only=True):
previous = beats[-1]
beats.append(BeatEvent(previous.time + period, previous.beat_id % previous.declared_numerator + 1,
previous.declared_numerator, previous.denominator))
intervals = {}
for field in ("key", "structure", "chord"):
intervals = {field: interval_rows(events, field, duration) for field in ("key", "structure", "chord")}
intervals["key"] = [[start, end, normalize_key_name(key)] for start, end, key in intervals["key"]]
intervals["chord"] = correct_chord_rows(intervals["chord"], intervals["key"])
for field, rows in intervals.items():
intervals[field] = [[max(beats[0].time, start), min(beats[-1].time, end), value]
for start, end, value in interval_rows(events, field, duration)
for start, end, value in rows
if end > beats[0].time and start < beats[-1].time]
if not intervals["key"]:
raise AbcRebuildError("SheetSage2 did not decode a key for the ABC score.")
Expand Down Expand Up @@ -195,6 +198,31 @@ class MeasureGroup:

_FLAT_PITCH_NAMES = ("C", "Db", "D", "Eb", "E", "F", "Gb", "G", "Ab", "A", "Bb", "B")

# SheetSage2 chord_spelling_sheetsage2.py, revision 55bfe14: canonical keys
# and chord-tone offsets on the circle of fifths for the decoder's vocabulary.
_KEY_TONICS = {
"major": ("C", "Db", "D", "Eb", "E", "F", "F#", "G", "Ab", "A", "Bb", "B"),
"minor": ("C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "Bb", "B"),
}
_NATURAL_FIFTHS = {"C": 1, "D": 3, "E": 5, "F": 0, "G": 2, "A": 4, "B": 6}
_CHORD_TONE_FIFTHS = {
"maj": (0, 4, 1),
"min": (0, -3, 1),
"dim": (0, -3, -6),
"aug": (0, 4, 8),
"maj7": (0, 4, 1, 5),
"min7": (0, -3, 1, -2),
"7": (0, 4, 1, -2),
"hdim7": (0, -3, -6, -2),
"dim7": (0, -3, -6, -9),
"minmaj7": (0, -3, 1, 5),
"sus2": (0, 2, 1),
"sus4": (0, -1, 1),
"sus4(b7)": (0, -1, 1, -2),
"maj6": (0, 4, 1, 3),
"min6": (0, -3, 1, 3),
}

_ROOT_RE = re.compile(r"^(?P<letter>[A-G])(?P<accidental>#{0,2}|b{0,2})$")

_BASS_DEGREE_RE = re.compile(r"^(?P<accidental>#{0,2}|b{0,2})(?P<degree>[1-9]|1[0-3])$")
Expand Down Expand Up @@ -442,6 +470,44 @@ def _pitch_class(root: str) -> tuple[int, str, str]:
offset = accidental.count("#") - accidental.count("b")
return (_NATURAL_PITCH_CLASS[letter] + offset) % 12, letter, accidental

def normalize_key_name(key: str) -> str:
root, mode = key.split(":")
if mode not in _KEY_TONICS:
raise AbcRebuildError(f"Unsupported key mode {mode!r} in {key!r}")
return f"{_KEY_TONICS[mode][_pitch_class(root)[0]]}:{mode}"

def correct_chord_spelling(chord: str, key: str) -> str:
if chord in NO_CHORDS:
return chord
root, descriptor = chord.split(":", 1)
quality = descriptor.split("/", 1)[0]
if quality not in _CHORD_TONE_FIFTHS:
raise ChordSymbolError(f"Unsupported chord quality {quality!r} in {chord!r}")
tonic, mode = key.split(":")
scale = (_pitch_class(tonic)[0] - (9 if mode == "minor" else 0)) % 12
_, key_letter, key_accidental = _pitch_class(_KEY_TONICS["major"][scale])
key_position = _NATURAL_FIFTHS[key_letter] + 7 * (key_accidental.count("#") - key_accidental.count("b"))
root_pc = _pitch_class(root)[0]
candidates = []
for letter in _LETTERS:
accidental = (root_pc - _NATURAL_PITCH_CLASS[letter] + 6) % 12 - 6
position = _NATURAL_FIFTHS[letter] + accidental * 7
distances = [max(abs(position + offset - (key_position + 2)) - 3, 0)
for offset in _CHORD_TONE_FIFTHS[quality]]
# Weight the root twice and resolve ties in CDEFGAB order, as upstream.
score = sum(distances) + distances[0]
spelling = letter + ("#" * accidental if accidental >= 0 else "b" * -accidental)
candidates.append((score, spelling))
return f"{min(candidates, key=lambda candidate: candidate[0])[1]}:{descriptor}"

def correct_chord_rows(chords, keys):
if not keys:
return [list(row) for row in chords]
boundaries = [end for _, end, _ in keys[:-1]]
# Midpoint ties use the key on the left; edge chords use the first/last key.
return [[start, end, correct_chord_spelling(chord, keys[bisect_left(boundaries, (start + end) / 2)][2])]
for start, end, chord in chords]

def portable_pitch_name(root: str, *, preserve_double: bool = False) -> str:
pitch_class, _, accidental = _pitch_class(root)
if preserve_double or len(accidental) <= 1:
Expand Down
16 changes: 16 additions & 0 deletions tests-unit/assets_test/services/path_prefix_cases.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,19 @@ def prefix_case_paths(root: str) -> list[tuple[str, bool]]:

def expected_prefix_case_paths(root: str) -> set[str]:
return {os.path.abspath(path) for path, expected in prefix_case_paths(root) if expected}


def anchor_case_paths() -> list[tuple[str, str, bool]]:
"""(path, prefix, Path(path).is_relative_to(prefix)) across POSIX anchors.

abspath keeps exactly two leading slashes, and pathlib treats "//" as its own
anchor, so "//server/..." is not under "/". Three or more collapse to "/".
"""
return [
("//server/file", "/", False),
("//server/file", "//server", True),
("//server", "//", True),
("/x", "//", False),
("///x/y", "/", True),
("/x/y", "/", True),
]
38 changes: 38 additions & 0 deletions tests-unit/assets_test/services/test_path_prefix_matcher.py
Loading
Loading