|
| 1 | +from typing import List, Optional, Union, Dict, Any, Tuple |
| 2 | + |
| 3 | +from PIL import Image |
| 4 | +import numpy as np |
| 5 | + |
| 6 | +from .catalog import PathManager, LABEL_MAP_CATALOG |
| 7 | +from ..base_layoutmodel import BaseLayoutModel |
| 8 | +from ...elements import Rectangle, TextBlock, Layout |
| 9 | + |
| 10 | +from ...file_utils import is_effdet_available, is_torch_cuda_available |
| 11 | + |
| 12 | +if is_effdet_available(): |
| 13 | + import torch |
| 14 | + from effdet import create_model |
| 15 | + from effdet.data.transforms import ( |
| 16 | + IMAGENET_DEFAULT_MEAN, |
| 17 | + IMAGENET_DEFAULT_STD, |
| 18 | + transforms_coco_eval, |
| 19 | + ) |
| 20 | + |
| 21 | + |
| 22 | +class InputTransform: |
| 23 | + def __init__( |
| 24 | + self, |
| 25 | + image_size, |
| 26 | + mean=IMAGENET_DEFAULT_MEAN, |
| 27 | + std=IMAGENET_DEFAULT_STD, |
| 28 | + ): |
| 29 | + |
| 30 | + self.mean = mean |
| 31 | + self.std = std |
| 32 | + |
| 33 | + self.transform = transforms_coco_eval( |
| 34 | + image_size, |
| 35 | + interpolation="bilinear", |
| 36 | + use_prefetcher=True, |
| 37 | + fill_color="mean", |
| 38 | + mean=self.mean, |
| 39 | + std=self.std, |
| 40 | + ) |
| 41 | + |
| 42 | + self.mean_tensor = torch.tensor([x * 255 for x in mean]).view(1, 3, 1, 1) |
| 43 | + self.std_tensor = torch.tensor([x * 255 for x in std]).view(1, 3, 1, 1) |
| 44 | + |
| 45 | + def preprocess(self, image: Image) -> Tuple[torch.Tensor, Dict]: |
| 46 | + |
| 47 | + image = image.convert("RGB") |
| 48 | + image_info = {"img_size": image.size} |
| 49 | + |
| 50 | + input, image_info = self.transform(image, image_info) |
| 51 | + image_info = { |
| 52 | + key: torch.tensor(val).unsqueeze(0) for key, val in image_info.items() |
| 53 | + } |
| 54 | + |
| 55 | + input = torch.tensor(input).unsqueeze(0) |
| 56 | + input = input.float().sub_(self.mean_tensor).div_(self.std_tensor) |
| 57 | + |
| 58 | + return input, image_info |
| 59 | + |
| 60 | + |
| 61 | +class EfficientDetLayoutModel(BaseLayoutModel): |
| 62 | + """Create a EfficientDet-based Layout Detection Model |
| 63 | +
|
| 64 | + Args: |
| 65 | + config_path (:obj:`str`): |
| 66 | + The path to the configuration file. |
| 67 | + model_path (:obj:`str`, None): |
| 68 | + The path to the saved weights of the model. |
| 69 | + If set, overwrite the weights in the configuration file. |
| 70 | + Defaults to `None`. |
| 71 | + label_map (:obj:`dict`, optional): |
| 72 | + The map from the model prediction (ids) to real |
| 73 | + word labels (strings). If the config is from one of the supported |
| 74 | + datasets, Layout Parser will automatically initialize the label_map. |
| 75 | + Defaults to `None`. |
| 76 | + enforce_cpu(:obj:`bool`, optional): |
| 77 | + When set to `True`, it will enforce using cpu even if it is on a CUDA |
| 78 | + available device. |
| 79 | + extra_config (:obj:`dict`, optional): |
| 80 | + Extra configuration passed to the EfficientDet model |
| 81 | + configuration. Currently supported arguments: |
| 82 | + num_classes: specifying the number of classes for the models |
| 83 | + output_confidence_threshold: minmum object prediction confidence to retain |
| 84 | +
|
| 85 | + Examples:: |
| 86 | + >>> import layoutparser as lp |
| 87 | + >>> model = lp.EfficientDetLayoutModel("lp://PubLayNet/tf_efficientdet_d0/config") |
| 88 | + >>> model.detect(image) |
| 89 | +
|
| 90 | + """ |
| 91 | + |
| 92 | + DEPENDENCIES = ["effdet"] |
| 93 | + DETECTOR_NAME = "efficientdet" |
| 94 | + |
| 95 | + DEFAULT_OUTPUT_CONFIDENCE_THRESHOLD = 0.25 |
| 96 | + |
| 97 | + def __init__( |
| 98 | + self, |
| 99 | + config_path: str, |
| 100 | + model_path: str = None, |
| 101 | + label_map: Optional[Dict] = None, |
| 102 | + extra_config: Optional[Dict] = None, |
| 103 | + enforce_cpu: bool = False, |
| 104 | + device: str = None, |
| 105 | + ): |
| 106 | + |
| 107 | + if is_torch_cuda_available(): |
| 108 | + if device is None: |
| 109 | + device = "cuda" |
| 110 | + else: |
| 111 | + device = "cpu" |
| 112 | + self.device = device |
| 113 | + |
| 114 | + extra_config = extra_config if extra_config is not None else {} |
| 115 | + |
| 116 | + self._initialize_model(config_path, model_path, label_map, extra_config) |
| 117 | + |
| 118 | + self.output_confidence_threshold = extra_config.get( |
| 119 | + "output_confidence_threshold", self.DEFAULT_OUTPUT_CONFIDENCE_THRESHOLD |
| 120 | + ) |
| 121 | + |
| 122 | + self.preprocessor = InputTransform(self.config.image_size) |
| 123 | + |
| 124 | + def _initialize_model( |
| 125 | + self, |
| 126 | + config_path: str, |
| 127 | + model_path: Optional[str], |
| 128 | + label_map: Optional[Dict], |
| 129 | + extra_config: Optional[Dict], |
| 130 | + ): |
| 131 | + |
| 132 | + if config_path.startswith("lp://"): |
| 133 | + # If it's officially supported by layoutparser |
| 134 | + dataset_name, model_name = config_path.lstrip("lp://").split("/")[0:2] |
| 135 | + |
| 136 | + if label_map is None: |
| 137 | + label_map = LABEL_MAP_CATALOG[dataset_name] |
| 138 | + num_classes = len(label_map) |
| 139 | + |
| 140 | + if model_path is None: |
| 141 | + # Download the models when it model_path is not specified |
| 142 | + model_path = PathManager.get_local_path( |
| 143 | + self._reconstruct_path_with_detector_name( |
| 144 | + config_path.replace("config", "weight") |
| 145 | + ) |
| 146 | + ) |
| 147 | + |
| 148 | + self.model = create_model( |
| 149 | + model_name, |
| 150 | + num_classes=num_classes, |
| 151 | + bench_task="predict", |
| 152 | + pretrained=True, |
| 153 | + checkpoint_path=model_path, |
| 154 | + ) |
| 155 | + else: |
| 156 | + assert ( |
| 157 | + model_path is not None |
| 158 | + ), f"When the specified model is not layoutparser-based, you need to specify the model_path" |
| 159 | + |
| 160 | + assert ( |
| 161 | + label_map is not None or "num_classes" in extra_config |
| 162 | + ), "When the specified model is not layoutparser-based, you need to specify the label_map or add num_classes in the extra_config" |
| 163 | + |
| 164 | + model_name = config_path |
| 165 | + model_path = PathManager.get_local_path( |
| 166 | + model_path |
| 167 | + ) # It might be an https URL |
| 168 | + |
| 169 | + num_classes = len(label_map) if label_map else extra_config["num_classes"] |
| 170 | + |
| 171 | + self.model = create_model( |
| 172 | + model_name, |
| 173 | + num_classes=num_classes, |
| 174 | + bench_task="predict", |
| 175 | + pretrained=True, |
| 176 | + checkpoint_path=model_path, |
| 177 | + ) |
| 178 | + |
| 179 | + self.model.to(self.device) |
| 180 | + self.model.eval() |
| 181 | + self.config = self.model.config |
| 182 | + self.label_map = label_map if label_map is not None else {} |
| 183 | + |
| 184 | + def _reconstruct_path_with_detector_name(self, path: str) -> str: |
| 185 | + """This function will add the detector name (efficientdet) into the |
| 186 | + lp model config path to get the "canonical" model name. |
| 187 | +
|
| 188 | + Args: |
| 189 | + path (str): The given input path that might or might not contain the detector name. |
| 190 | +
|
| 191 | + Returns: |
| 192 | + str: a modified path that contains the detector name. |
| 193 | + """ |
| 194 | + if path.startswith("lp://"): # TODO: Move "lp://" to a constant |
| 195 | + model_name = path[len("lp://") :] |
| 196 | + model_name_segments = model_name.split("/") |
| 197 | + if ( |
| 198 | + len(model_name_segments) == 3 |
| 199 | + and self.DETECTOR_NAME not in model_name_segments |
| 200 | + ): |
| 201 | + return "lp://" + self.DETECTOR_NAME + "/" + path[len("lp://") :] |
| 202 | + return path |
| 203 | + |
| 204 | + def detect(self, image: Union["np.ndarray", "Image.Image"]): |
| 205 | + |
| 206 | + image = self.image_loader(image) |
| 207 | + |
| 208 | + model_inputs, image_info = self.preprocessor.preprocess(image) |
| 209 | + |
| 210 | + model_outputs = self.model( |
| 211 | + model_inputs.to(self.device), |
| 212 | + {key: val.to(self.device) for key, val in image_info.items()}, |
| 213 | + ) |
| 214 | + |
| 215 | + layout = self.gather_output(model_outputs) |
| 216 | + return layout |
| 217 | + |
| 218 | + def gather_output(self, model_outputs: torch.Tensor) -> Layout: |
| 219 | + |
| 220 | + model_outputs = model_outputs.cpu().detach() |
| 221 | + box_predictions = Layout() |
| 222 | + |
| 223 | + for index, sample in enumerate(model_outputs): |
| 224 | + sample[:, 2] -= sample[:, 0] |
| 225 | + sample[:, 3] -= sample[:, 1] |
| 226 | + |
| 227 | + for det in sample: |
| 228 | + |
| 229 | + score = float(det[4]) |
| 230 | + pred_cat = int(det[5]) |
| 231 | + x, y, w, h = det[0:4].tolist() |
| 232 | + |
| 233 | + if ( |
| 234 | + score < self.output_confidence_threshold |
| 235 | + ): # stop when below this threshold, scores in descending order |
| 236 | + break |
| 237 | + |
| 238 | + box_predictions.append( |
| 239 | + TextBlock( |
| 240 | + block=Rectangle(x, y, w + x, h + y), |
| 241 | + score=score, |
| 242 | + id=index, |
| 243 | + type=self.label_map.get(pred_cat, pred_cat), |
| 244 | + ) |
| 245 | + ) |
| 246 | + |
| 247 | + return box_predictions |
| 248 | + |
| 249 | + def image_loader(self, image: Union["np.ndarray", "Image.Image"]): |
| 250 | + |
| 251 | + # Convert cv2 Image Input |
| 252 | + if isinstance(image, np.ndarray): |
| 253 | + # In this case, we assume the image is loaded by cv2 |
| 254 | + # and the channel order is BGR |
| 255 | + image = image[..., ::-1] |
| 256 | + image = Image.fromarray(image, mode="RGB") |
| 257 | + |
| 258 | + return image |
0 commit comments