Thanks to visit codestin.com
Credit goes to github.com

Skip to content

TensorRT Support #45

Description

@ntoxeg

It would be beneficial for performance to be able to apply TensorRT optimizations to the model.
Sadly, applying them like this

model = torch_tensorrt.MutableTorchTensorRTModule(
    model,
    strict=False,
    allow_complex_guards_as_runtime_asserts=True,
    enabled_precisions={torch.bfloat16},
    truncate_double=True,
    immutable_weights=False,
    offload_module_to_cpu=True,
)

I get

Traceback (most recent call last):
  File "/home/greg/dream-inference/simple_test.py", line 33, in <module>
    output = model.diffusion_generate(
             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/greg/.cache/huggingface/modules/transformers_modules/Dream-org/Dream-v0-Instruct-7B/05334cb9faaf763692dcf9d8737c642be2b2a6ae/generation_utils.py", line 355, in diffusion_generate
    result = self._sample(
             ^^^^^^^^^^^^^
  File "/home/greg/.cache/huggingface/modules/transformers_modules/Dream-org/Dream-v0-Instruct-7B/05334cb9faaf763692dcf9d8737c642be2b2a6ae/generation_utils.py", line 411, in _sample
    logits = self(x, attention_mask, tok_idx).logits
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.py", line 516, in __call__
    return self.forward(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.py", line 479, in forward
    self.compile()
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.py", line 360, in compile
    self.exp_program = torch.export.export(
                       ^^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/__init__.py", line 360, in export
    return _export(
           ^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 1100, in wrapper
    raise e
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 1066, in wrapper
    ep = fn(*args, **kwargs)
         ^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/exported_program.py", line 121, in wrapper
    return fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 2120, in _export
    ep = _export_for_training(
         ^^^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 1100, in wrapper
    raise e
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 1066, in wrapper
    ep = fn(*args, **kwargs)
         ^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/exported_program.py", line 121, in wrapper
    return fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 1983, in _export_for_training
    export_artifact = export_func(  # type: ignore[operator]
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 1352, in _strict_export_lower_to_aten_ir
    gm_torch_level = _export_to_torch_ir(
                     ^^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 740, in _export_to_torch_ir
    gm_torch_level, _ = torch._dynamo.export(
                        ^^^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py", line 1677, in inner
    result_traced = opt_f(*args, **kwargs)
                    ^^^^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py", line 659, in _fn
    raise e.with_traceback(None) from None
torch._dynamo.exc.Unsupported: call_id with args (GetAttrVariable(ConstantVariable(NoneType: None), __init__),)

from user code:
   File "/home/greg/.cache/huggingface/modules/transformers_modules/Dream-org/Dream-v0-Instruct-7B/05334cb9faaf763692dcf9d8737c642be2b2a6ae/modeling_dream.py", line 794, in forward
    outputs = self.model(
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/greg/.cache/huggingface/modules/transformers_modules/Dream-org/Dream-v0-Instruct-7B/05334cb9faaf763692dcf9d8737c642be2b2a6ae/modeling_dream.py", line 668, in forward
    past_key_values = DynamicCache()
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/_dynamo/polyfills/__init__.py", line 156, in instantiate_user_defined_class_object
    obj.__init__(*args, **kwargs)
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/transformers/cache_utils.py", line 1300, in __init__
    super().__init__(layer_classes=DynamicLayer, *args, **kwargs)
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/transformers/cache_utils.py", line 1080, in __init__
    processor_kwargs, kwargs = parse_processor_args(processor_class, kwargs)
  File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/transformers/cache_utils.py", line 1876, in parse_processor_args
    params = list(inspect.signature(processor_class.__init__).parameters)[2:]
  File "/home/greg/.local/share/uv/python/cpython-3.12.11-linux-x86_64-gnu/lib/python3.12/inspect.py", line 3348, in signature
    return Signature.from_callable(obj, follow_wrapped=follow_wrapped,
  File "/home/greg/.local/share/uv/python/cpython-3.12.11-linux-x86_64-gnu/lib/python3.12/inspect.py", line 3085, in from_callable
    return _signature_from_callable(obj, sigcls=cls,
  File "/home/greg/.local/share/uv/python/cpython-3.12.11-linux-x86_64-gnu/lib/python3.12/inspect.py", line 2538, in _signature_from_callable
    obj = unwrap(obj, stop=(lambda f: hasattr(f, "__signature__")
  File "/home/greg/.local/share/uv/python/cpython-3.12.11-linux-x86_64-gnu/lib/python3.12/inspect.py", line 773, in unwrap
    memo = {id(f): f}

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Fields

    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions