Skip to content

vllm.utils.gc_utils

Classes:

  • GCDebugConfig –

    Config for GC Debugger.

  • GCDebugger –

    Debugger for GC which logs helpful information for GC understanding.

Functions:

GCDebugConfig

Config for GC Debugger. - 0: disable GC debugger - 1: enable GC debugger with gc.collect elapsed times - '{"top_objects":5}': enable GC debugger with top 5 collected objects

Source code in vllm/utils/gc_utils.py 
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
class GCDebugConfig:
    """Config for GC Debugger.
    - 0: disable GC debugger
    - 1: enable GC debugger with gc.collect elapsed times
    - '{"top_objects":5}': enable GC debugger with top 5 collected objects
    """

    def __init__(self, gc_debug_conf: str | None = None) -> None:
        self.enabled: bool = False
        self.top_objects: int = -1

        if not gc_debug_conf or gc_debug_conf == "0":
            pass
        elif gc_debug_conf == "1":
            self.enabled = True
        else:
            try:
                json_conf = json.loads(gc_debug_conf)
                self.enabled = True
                self.top_objects = json_conf.get("top_objects", -1)
            except Exception:
                self.enabled = False
                logger.error("Failed to parse VLLM_GC_DEBUG(%s)", envs.VLLM_GC_DEBUG)
        logger.debug("GC Debug Config. %s", str(self))

    def __repr__(self) -> str:
        return f"enabled:{self.enabled},top_objects:{self.top_objects}"

GCDebugger

Debugger for GC which logs helpful information for GC understanding. To enable, you should call maybe_attach_gc_debug_callback in the process.

Methods:

  • handle –

    Handles a GC event (e.g. GC start or GC finish).

Source code in vllm/utils/gc_utils.py 
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
class GCDebugger:
    """Debugger for GC which logs helpful information for GC understanding.
    To enable, you should call maybe_attach_gc_debug_callback in the process.
    """

    def __init__(self, config: GCDebugConfig) -> None:
        self.config = config
        # Start time in micro second of this GC cycle
        self.start_time_ns: int = time.monotonic_ns()
        self.num_objects: int = 0
        # If config.top_objects is positive,
        # compute top collected objects by object types
        self.gc_top_collected_objects: str = ""

    def handle(self, phase: str, info: dict[str, int]) -> None:
        """Handles a GC event (e.g. GC start or GC finish)."""
        generation = info.get("generation")
        if generation is None:
            return
        if phase == "start":
            # Before GC started, record GC start time
            # and top collected objects
            self.start_time_ns = time.monotonic_ns()
            objects = gc.get_objects(generation)
            self.num_objects = len(objects)
            self.gc_top_collected_objects = _compute_top_gc_collected_objects(
                objects, self.config.top_objects
            )
        elif phase == "stop":
            # After GC finished, Record GC elapsed time and
            # optionally top collected objects
            elapsed_ms = (time.monotonic_ns() - self.start_time_ns) / 1e6
            logger.info(
                "GC took %.3fms to complete. "
                "Collected %s objects (out of %d) in GC generation %d.%s",
                elapsed_ms,
                str(info.get("collected", "?")),
                self.num_objects,
                generation,
                (
                    f" Top collected objects: \n{self.gc_top_collected_objects}"
                    if self.gc_top_collected_objects
                    else ""
                ),
            )

handle(phase, info)

Handles a GC event (e.g. GC start or GC finish).

Source code in vllm/utils/gc_utils.py 
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
def handle(self, phase: str, info: dict[str, int]) -> None:
    """Handles a GC event (e.g. GC start or GC finish)."""
    generation = info.get("generation")
    if generation is None:
        return
    if phase == "start":
        # Before GC started, record GC start time
        # and top collected objects
        self.start_time_ns = time.monotonic_ns()
        objects = gc.get_objects(generation)
        self.num_objects = len(objects)
        self.gc_top_collected_objects = _compute_top_gc_collected_objects(
            objects, self.config.top_objects
        )
    elif phase == "stop":
        # After GC finished, Record GC elapsed time and
        # optionally top collected objects
        elapsed_ms = (time.monotonic_ns() - self.start_time_ns) / 1e6
        logger.info(
            "GC took %.3fms to complete. "
            "Collected %s objects (out of %d) in GC generation %d.%s",
            elapsed_ms,
            str(info.get("collected", "?")),
            self.num_objects,
            generation,
            (
                f" Top collected objects: \n{self.gc_top_collected_objects}"
                if self.gc_top_collected_objects
                else ""
            ),
        )

_compute_detailed_type(o)

Detailed object type.

TODO(Jialin): Further enhance the detailed type with element types for easier debugging. We tried but occasionally it would run into signals which kills the engine.

Source code in vllm/utils/gc_utils.py 
188
189
190
191
192
193
194
195
196
197
198
199
200
def _compute_detailed_type(o: Any) -> str:
    """Detailed object type.

    TODO(Jialin): Further enhance the detailed type with element types for
    easier debugging. We tried but occasionally it would run into signals
    which kills the engine.
    """
    size_str: str = ""
    # Object doesn't support len() - this can happen with type objects
    # or other objects that don't implement __len__ properly
    with suppress(Exception):
        size_str = f"(size:{len(o)})"
    return f"{str(type(o))}{size_str}"

_compute_top_gc_collected_objects(objects, top)

Group collected objects by types.

Source code in vllm/utils/gc_utils.py 
203
204
205
206
207
208
209
210
211
def _compute_top_gc_collected_objects(objects: list[Any], top: int) -> str:
    """Group collected objects by types."""
    if top <= 0:
        return ""
    object_types = [_compute_detailed_type(o) for o in objects]
    return "\n".join(
        f"{count:>5}:{object_type}"
        for object_type, count in Counter(object_types).most_common(top)
    )

freeze_gc_for_cudagraph_capture()

Freeze and disable gc for the duration of bulk CUDA graph capture.

A gc cycle during stream capture can invalidate the captured graph, e.g. a finalized Triton kernel unloads its module. Opt out with VLLM_ENABLE_CUDAGRAPH_GC=1.

Source code in vllm/utils/gc_utils.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
@contextmanager
def freeze_gc_for_cudagraph_capture():
    """Freeze and disable gc for the duration of bulk CUDA graph capture.

    A gc cycle during stream capture can invalidate the captured graph, e.g.
    a finalized Triton kernel unloads its module. Opt out with
    VLLM_ENABLE_CUDAGRAPH_GC=1.
    """
    gc_was_enabled = gc.isenabled()
    gc.collect()
    should_freeze = not envs.VLLM_ENABLE_CUDAGRAPH_GC
    if should_freeze:
        gc.freeze()
        gc.disable()
    try:
        yield
    finally:
        if should_freeze:
            try:
                gc.unfreeze()
                gc.collect()
            finally:
                if gc_was_enabled:
                    gc.enable()
                else:
                    gc.disable()

freeze_gc_heap()

Freeze all objects tracked by the garbage collector. It should be invoked after server init / warmup, to reduce GC overhead from static objects during serving time.

Source code in vllm/utils/gc_utils.py 
122
123
124
125
126
127
128
129
130
131
132
133
def freeze_gc_heap() -> None:
    """Freeze all objects tracked by the garbage collector. It should be invoked
    after server init / warmup, to reduce GC overhead from static objects
    during serving time.
    """
    # Ensure all static objects are pushed down to the oldest generation for
    # freeze
    gc.collect(0)
    gc.collect(1)
    gc.collect(2)
    # Freeze all GC tracked objects
    gc.freeze()

gc_untrack(obj)

Hide a container from the cyclic GC, so that collections skip its elements. Reference counting still frees it.

Only for containers that can never be part of a reference cycle (e.g. a list of atomic values, or of objects that never refer back to it): a cycle through an untracked container is never collected. Dicts re-track themselves when a tracked value is inserted, so this suits lists. A no-op for objects the GC does not track (e.g. arrays, empty tuples).

Source code in vllm/utils/gc_utils.py 
144
145
146
147
148
149
150
151
152
153
154
155
156
def gc_untrack(obj: _T) -> _T:
    """Hide a container from the cyclic GC, so that collections skip its
    elements. Reference counting still frees it.

    Only for containers that can never be part of a reference cycle (e.g. a
    list of atomic values, or of objects that never refer back to it): a cycle
    through an untracked container is never collected. Dicts re-track
    themselves when a tracked value is inserted, so this suits lists.
    A no-op for objects the GC does not track (e.g. arrays, empty tuples).
    """
    if gc.is_tracked(obj):
        _PyObject_GC_UnTrack(obj)
    return obj

maybe_attach_gc_debug_callback()

Attached a callback for GC debug when VLLM_GC_DEBUG is enabled.

Source code in vllm/utils/gc_utils.py 
176
177
178
179
180
181
182
183
184
185
def maybe_attach_gc_debug_callback() -> None:
    """Attached a callback for GC debug when VLLM_GC_DEBUG is enabled."""
    config = GCDebugConfig(envs.VLLM_GC_DEBUG)
    if config.enabled:
        debugger: GCDebugger = GCDebugger(config)

        def gc_callback(phase: str, info: dict[str, int]) -> None:
            debugger.handle(phase, info)

        gc.callbacks.append(gc_callback)

untrack_long_lists(lists, min_len=256)

gc_untrack the lists with at least min_len elements, skipping None.

For short-lived lists, which a young GC pass would scan at most once or twice: untracking costs ~0.2us per call against ~20ns per element scanned, so it only pays off for long ones.

Source code in vllm/utils/gc_utils.py 
164
165
166
167
168
169
170
171
172
173
def untrack_long_lists(lists: Iterable[list | None], min_len: int = 256) -> None:
    """`gc_untrack` the lists with at least `min_len` elements, skipping None.

    For short-lived lists, which a young GC pass would scan at most once or
    twice: untracking costs ~0.2us per call against ~20ns per element scanned,
    so it only pays off for long ones.
    """
    for lst in lists:
        if lst is not None and len(lst) >= min_len:
            gc_untrack(lst)

untracked_list()

An empty list that the cyclic GC never traverses (see gc_untrack).

Source code in vllm/utils/gc_utils.py 
159
160
161
def untracked_list() -> list:
    """An empty list that the cyclic GC never traverses (see `gc_untrack`)."""
    return gc_untrack([])