Skip to content

Commit

Permalink
[GPTQ] fix layerwise targets (#36)
Browse files Browse the repository at this point in the history
  • Loading branch information
Sara Adkins authored Jul 24, 2024
1 parent 07c1fd7 commit 29cb10d
Showing 1 changed file with 4 additions and 3 deletions.
7 changes: 4 additions & 3 deletions src/llmcompressor/modifiers/quantization/gptq/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ class GPTQModifier(Modifier):

sequential_update: Optional[bool] = False
targets: Union[str, List[str], None] = None
sequential_targets: Union[str, List[str], None] = None
block_size: int = 128
quantize: Union[bool, Dict] = True
dampening_frac: Optional[float] = 0.01
Expand Down Expand Up @@ -177,11 +178,11 @@ def on_initialize(self, state: "State", **kwargs) -> bool:
modifiable_model = state.model
calibration_dataloader = state.data.calib

if self.targets is None:
if self.sequential_targets is None:
# if no targets are provided, default to the modules that shouldn't be
# split by FSDP. For Transformers models this is equivalent to the
# decoder layers (ie LlamaDecoderLayer)
self.targets = get_no_split_params(modifiable_model)
self.sequential_targets = get_no_split_params(modifiable_model)

self.initialize_compression(modifiable_model, calibration_dataloader)
self.apply_compression(calibration_dataloader)
Expand Down Expand Up @@ -215,7 +216,7 @@ def compressible_layers(self) -> Dict:
f"{type(self.model)} instead"
)

return get_layers(self.targets, self.model)
return get_layers(self.sequential_targets, self.model)

def initialize_compression(
self,
Expand Down

0 comments on commit 29cb10d

Please sign in to comment.