{ "id": "466144a71db1", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/", "source_excel_row": 1601, "source_url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/", "is_paid_flag": "", "images": [ { "status": "ok", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/yolo.png", "bytes": 947849, "reused": false, "content_type": "image/png", "local_path": "attachments/466144a71db1/images/body-7b981ee4ba3f0de9c69e.png", "src": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/yolo.png", "alt": "", "downloaded": true }, { "status": "ok", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/yolo-network-architecture.png", "bytes": 193831, "reused": false, "content_type": "image/png", "local_path": "attachments/466144a71db1/images/body-d2d51b39b0c71b0793d1.png", "src": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/yolo-network-architecture.png", "alt": "", "downloaded": true }, { "status": "ok", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/yolo-responsible-predictor.png", "bytes": 141158, "reused": false, "content_type": "image/png", "local_path": "attachments/466144a71db1/images/body-f004873880c2663f6f27.png", "src": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/yolo-responsible-predictor.png", "alt": "", "downloaded": true }, { "status": "ok", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/SSD-architecture.png", "bytes": 285879, "reused": false, "content_type": "image/png", "local_path": "attachments/466144a71db1/images/body-b686cd210511644b9482.png", "src": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/SSD-architecture.png", "alt": "", "downloaded": true }, { "status": "ok", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/SSD-framework.png", "bytes": 862890, "reused": false, "content_type": "image/png", "local_path": "attachments/466144a71db1/images/body-37c5aefd5995be6d0a22.png", "src": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/SSD-framework.png", "alt": "", "downloaded": true }, { "status": "ok", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/SSD-box-scales.png", "bytes": 155905, "reused": false, "content_type": "image/png", "local_path": "attachments/466144a71db1/images/body-4e81399abad9997ff29c.png", "src": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/SSD-box-scales.png", "alt": "", "downloaded": true }, { "status": "ok", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/yolov2-loc-prediction.png", "bytes": 120909, "reused": false, "content_type": "image/png", "local_path": "attachments/466144a71db1/images/body-ac3a14a3f394f51725f0.png", "src": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/yolov2-loc-prediction.png", "alt": "", "downloaded": true }, { "status": "ok", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/word-tree.png", "bytes": 1623681, "reused": false, "content_type": "image/png", "local_path": "attachments/466144a71db1/images/body-bf58d167b991ee5982f3.png", "src": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/word-tree.png", "alt": "", "downloaded": true }, { "status": "ok", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/focal-loss.png", "bytes": 328470, "reused": false, "content_type": "image/png", "local_path": "attachments/466144a71db1/images/body-7b68e660a804f6fdedfc.png", "src": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/focal-loss.png", "alt": "", "downloaded": true }, { "status": "ok", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/focal-loss-weights.png", "bytes": 77640, "reused": false, "content_type": "image/png", "local_path": "attachments/466144a71db1/images/body-2a4ecbef4a9439ed154d.png", "src": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/focal-loss-weights.png", "alt": "", "downloaded": true }, { "status": "ok", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/featurized-image-pyramid.png", "bytes": 310680, "reused": false, "content_type": "image/png", "local_path": "attachments/466144a71db1/images/body-955d0dc9f36806474855.png", "src": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/featurized-image-pyramid.png", "alt": "", "downloaded": true }, { "status": "ok", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/retina-net.png", "bytes": 905917, "reused": false, "content_type": "image/png", "local_path": "attachments/466144a71db1/images/body-2098115c6b769573d03c.png", "src": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/retina-net.png", "alt": "", "downloaded": true }, { "status": "ok", "url": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/yolov3-perf.png", "bytes": 332323, "reused": false, "content_type": "image/png", "local_path": "attachments/466144a71db1/images/body-90988a475dbc99affc49.png", "src": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/yolov3-perf.png", "alt": "", "downloaded": true } ], "videos": [], "body_markdown": "In\nPart 3\n, we have reviewed models in the R-CNN family. All of them are region-based object detection algorithms. They can achieve high accuracy but could be too slow for certain applications such as autonomous driving. In Part 4, we only focus on fast object detection models, including SSD, RetinaNet, and models in the YOLO family.\nLinks to all the posts in the series:\n[\nPart 1\n]\n[\nPart 2\n]\n[\nPart 3\n]\n[\nPart 4\n].\nTwo-stage vs One-stage Detectors\n#\nModels in the R-CNN family are all region-based. The detection happens in two stages: (1) First, the model proposes a set of regions of interests by select search or regional proposal network. The proposed regions are sparse as the potential bounding box candidates can be infinite. (2) Then a classifier only processes the region candidates.\nThe other different approach skips the region proposal stage and runs detection directly over a dense sampling of possible locations. This is how a one-stage object detection algorithm works. This is faster and simpler, but might potentially drag down the performance a bit.\nAll the models introduced in this post are one-stage detectors.\nYOLO: You Only Look Once\n#\nThe\nYOLO\nmodel (\n“You Only Look Once”\n;\nRedmon et al., 2016\n) is the very first attempt at building a fast real-time object detector. Because YOLO does not undergo the region proposal step and only predicts over a limited number of bounding boxes, it is able to do inference super fast.\nWorkflow\n#\nPre-train\na CNN network on image classification task.\nSplit an image into $S \\times S$ cells. If an object’s center falls into a cell, that cell is “responsible” for detecting the existence of that object. Each cell predicts (a) the location of $B$ bounding boxes, (b) a confidence score, and (c) a probability of object class conditioned on the existence of an object in the bounding box.\nThe\ncoordinates\nof bounding box are defined by a tuple of 4 values, (center x-coord, center y-coord, width, height) — $(x, y, w, h)$, where $x$ and $y$ are set to be offset of a cell location. Moreover, $x$, $y$, $w$ and $h$ are normalized by the image width and height, and thus all between (0, 1].\nA\nconfidence score\nindicates the likelihood that the cell contains an object:\nPr(containing an object) x IoU(pred, truth)\n; where\nPr\n= probability and\nIoU\n= interaction under union.\nIf the cell contains an object, it predicts a\nprobability\nof this object belonging to every class $C_i, i=1, \\dots, K$:\nPr(the object belongs to the class C_i | containing an object)\n. At this stage, the model only predicts one set of class probabilities per cell, regardless of the number of bounding boxes, $B$.\nIn total, one image contains $S \\times S \\times B$ bounding boxes, each box corresponding to 4 location predictions, 1 confidence score, and K conditional probabilities for object classification. The total prediction values for one image is $S \\times S \\times (5B + K)$, which is the tensor shape of the final conv layer of the model.\nThe final layer of the pre-trained CNN is modified to output a prediction tensor of size $S \\times S \\times (5B + K)$.\nThe workflow of YOLO model. (Image source:\noriginal paper\n)\nNetwork Architecture\n#\nThe base model is similar to\nGoogLeNet\nwith inception module replaced by 1x1 and 3x3 conv layers. The final prediction of shape $S \\times S \\times (5B + K)$ is produced by two fully connected layers over the whole conv feature map.\nThe network architecture of YOLO.\nLoss Function\n#\nThe loss consists of two parts, the\nlocalization loss\nfor bounding box offset prediction and the\nclassification loss\nfor conditional class probabilities. Both parts are computed as the sum of squared errors. Two scale parameters are used to control how much we want to increase the loss from bounding box coordinate predictions ($\\lambda_\\text{coord}$) and how much we want to decrease the loss of confidence score predictions for boxes without objects ($\\lambda_\\text{noobj}$). Down-weighting the loss contributed by background boxes is important as most of the bounding boxes involve no instance. In the paper, the model sets $\\lambda_\\text{coord} = 5$ and $\\lambda_\\text{noobj} = 0.5$.\n$$\n\\begin{aligned}\n\\mathcal{L}_\\text{loc} &= \\lambda_\\text{coord} \\sum_{i=0}^{S^2} \\sum_{j=0}^B \\mathbb{1}_{ij}^\\text{obj} [(x_i - \\hat{x}_i)^2 + (y_i - \\hat{y}_i)^2 + (\\sqrt{w_i} - \\sqrt{\\hat{w}_i})^2 + (\\sqrt{h_i} - \\sqrt{\\hat{h}_i})^2 ] \\\\\n\\mathcal{L}_\\text{cls} &= \\sum_{i=0}^{S^2} \\sum_{j=0}^B \\big( \\mathbb{1}_{ij}^\\text{obj} + \\lambda_\\text{noobj} (1 - \\mathbb{1}_{ij}^\\text{obj})\\big) (C_{ij} - \\hat{C}_{ij})^2 + \\sum_{i=0}^{S^2} \\sum_{c \\in \\mathcal{C}} \\mathbb{1}_i^\\text{obj} (p_i(c) - \\hat{p}_i(c))^2\\\\\n\\mathcal{L} &= \\mathcal{L}_\\text{loc} + \\mathcal{L}_\\text{cls}\n\\end{aligned}\n$$\nNOTE: In the original YOLO paper, the loss function uses $C_i$ instead of $C_{ij}$ as confidence score. I made the correction based on my own understanding, since every bounding box should have its own confidence score. Please kindly let me if you do not agree. Many thanks.\nwhere,\n$\\mathbb{1}_i^\\text{obj}$: An indicator function of whether the cell i contains an object.\n$\\mathbb{1}_{ij}^\\text{obj}$: It indicates whether the j-th bounding box of the cell i is “responsible” for the object prediction (see Fig. 3).\n$C_{ij}$: The confidence score of cell i,\nPr(containing an object) * IoU(pred, truth)\n.\n$\\hat{C}_{ij}$: The predicted confidence score.\n$\\mathcal{C}$: The set of all classes.\n$p_i(c)$: The conditional probability of whether cell i contains an object of class $c \\in \\mathcal{C}$.\n$\\hat{p}_i(c)$: The predicted conditional class probability.\nAt one location, in cell i, the model proposes B bounding box candidates and the one that has highest overlap with the ground truth is the \"responsible\" predictor.\nThe loss function only penalizes classification error if an object is present in that grid cell, $\\mathbb{1}_i^\\text{obj} = 1$. It also only penalizes bounding box coordinate error if that predictor is “responsible” for the ground truth box, $\\mathbb{1}_{ij}^\\text{obj} = 1$.\nAs a one-stage object detector, YOLO is super fast, but it is not good at recognizing irregularly shaped objects or a group of small objects due to a limited number of bounding box candidates.\nSSD: Single Shot MultiBox Detector\n#\nThe\nSingle Shot Detector\n(\nSSD\n;\nLiu et al, 2016\n) is one of the first attempts at using convolutional neural network’s pyramidal feature hierarchy for efficient detection of objects of various sizes.\nImage Pyramid\n#\nSSD uses the\nVGG-16\nmodel pre-trained on ImageNet as its base model for extracting useful image features.\nOn top of VGG16, SSD adds several conv feature layers of decreasing sizes. They can be seen as a\npyramid representation\nof images at different scales. Intuitively large fine-grained feature maps at earlier levels are good at capturing small objects and small coarse-grained feature maps can detect large objects well. In SSD, the detection happens in every pyramidal layer, targeting at objects of various sizes.\nThe model architecture of SSD.\nWorkflow\n#\nUnlike YOLO, SSD does not split the image into grids of arbitrary size but predicts offset of predefined\nanchor boxes\n(this is called “default boxes” in the paper) for every location of the feature map. Each box has a fixed size and position relative to its corresponding cell. All the anchor boxes tile the whole feature map in a convolutional manner.\nFeature maps at different levels have different receptive field sizes. The anchor boxes on different levels are rescaled so that one feature map is only responsible for objects at one particular scale. For example, in Fig. 5 the dog can only be detected in the 4x4 feature map (higher level) while the cat is just captured by the 8x8 feature map (lower level).\nThe SSD framework. (a) The training data contains images and ground truth boxes for every object. (b) In a fine-grained feature maps (8 x 8), the anchor boxes of different aspect ratios correspond to smaller area of the raw input. (c) In a coarse-grained feature map (4 x 4), the anchor boxes cover larger area of the raw input. (Image source:\noriginal paper\n)\nThe width, height and the center location of an anchor box are all normalized to be (0, 1). At a location $(i, j)$ of the $\\ell$-th feature layer of size $m \\times n$, $i=1,\\dots,n, j=1,\\dots,m$, we have a unique linear scale proportional to the layer level and 5 different box aspect ratios (width-to-height ratios), in addition to a special scale (why we need this? the paper didn’t explain. maybe just a heuristic trick) when the aspect ratio is 1. This gives us 6 anchor boxes in total per feature cell.\n$$\n\\begin{aligned}\n\\text{level index: } &\\ell = 1, \\dots, L \\\\\n\\text{scale of boxes: } &s_\\ell = s_\\text{min} + \\frac{s_\\text{max} - s_\\text{min}}{L - 1} (\\ell - 1) \\\\\n\\text{aspect ratio: } &r \\in \\{1, 2, 3, 1/2, 1/3\\}\\\\\n\\text{additional scale: } & s'_\\ell = \\sqrt{s_\\ell s_{\\ell + 1}} \\text{ when } r = 1 \\text{thus, 6 boxes in total.}\\\\\n\\text{width: } &w_\\ell^r = s_\\ell \\sqrt{r} \\\\\n\\text{height: } &h_\\ell^r = s_\\ell / \\sqrt{r} \\\\\n\\text{center location: } & (x^i_\\ell, y^j_\\ell) = (\\frac{i+0.5}{m}, \\frac{j+0.5}{n})\n\\end{aligned}\n$$\nAn example of how the anchor box size is scaled up with the layer index $\\ell$ for $L=6, s\\_\\text{min} = 0.2, s\\_\\text{max} = 0.9$. Only the boxes of aspect ratio $r=1$ are illustrated.\nAt every location, the model outputs 4 offsets and $c$ class probabilities by applying a $3 \\times 3 \\times p$ conv filter (where $p$ is the number of channels in the feature map) for every one of $k$ anchor boxes. Therefore, given a feature map of size $m \\times n$, we need $kmn(c+4)$ prediction filters.\nLoss Function\n#\nSame as YOLO, the loss function is the sum of a localization loss and a classification loss.\n$\\mathcal{L} = \\frac{1}{N}(\\mathcal{L}_\\text{cls} + \\alpha \\mathcal{L}_\\text{loc})$\nwhere $N$ is the number of matched bounding boxes and $\\alpha$ balances the weights between two losses, picked by cross validation.\nThe\nlocalization loss\nis a\nsmooth L1 loss\nbetween the predicted bounding box correction and the true values. The coordinate correction transformation is same as what\nR-CNN\ndoes in\nbounding box regression\n.\n$$\n\\begin{aligned}\n\\mathcal{L}_\\text{loc} &= \\sum_{i,j} \\sum_{m\\in\\{x, y, w, h\\}} \\mathbb{1}_{ij}^\\text{match}\n L_1^\\text{smooth}(d_m^i - t_m^j)^2\\\\\nL_1^\\text{smooth}(x) &= \\begin{cases}\n 0.5 x^2 & \\text{if } \\vert x \\vert < 1\\\\\n \\vert x \\vert - 0.5 & \\text{otherwise}\n\\end{cases} \\\\\nt^j_x &= (g^j_x - p^i_x) / p^i_w \\\\\nt^j_y &= (g^j_y - p^i_y) / p^i_h \\\\\nt^j_w &= \\log(g^j_w / p^i_w) \\\\\nt^j_h &= \\log(g^j_h / p^i_h)\n\\end{aligned}\n$$\nwhere $\\mathbb{1}_{ij}^\\text{match}$ indicates whether the $i$-th bounding box with coordinates $(p^i_x, p^i_y, p^i_w, p^i_h)$ is matched to the $j$-th ground truth box with coordinates $(g^j_x, g^j_y, g^j_w, g^j_h)$ for any object. $d^i_m, m\\in\\{x, y, w, h\\}$ are the predicted correction terms. See\nthis\nfor how the transformation works.\nThe\nclassification loss\nis a softmax loss over multiple classes (\nsoftmax_cross_entropy_with_logits\nin tensorflow):\n$$\n\\mathcal{L}_\\text{cls} = -\\sum_{i \\in \\text{pos}} \\mathbb{1}_{ij}^k \\log(\\hat{c}_i^k) - \\sum_{i \\in \\text{neg}} \\log(\\hat{c}_i^0)\\text{, where }\\hat{c}_i^k = \\text{softmax}(c_i^k)\n$$\nwhere $\\mathbb{1}_{ij}^k$ indicates whether the $i$-th bounding box and the $j$-th ground truth box are matched for an object in class $k$. $\\text{pos}$ is the set of matched bounding boxes ($N$ items in total) and $\\text{neg}$ is the set of negative examples. SSD uses\nhard negative mining\nto select easily misclassified negative examples to construct this $\\text{neg}$ set: Once all the anchor boxes are sorted by objectiveness confidence score, the model picks the top candidates for training so that neg:pos is at most 3:1.\nYOLOv2 / YOLO9000\n#\nYOLOv2\n(\nRedmon & Farhadi, 2017\n) is an enhanced version of YOLO.\nYOLO9000\nis built on top of YOLOv2 but trained with joint dataset combining the COCO detection dataset and the top 9000 classes from ImageNet.\nYOLOv2 Improvement\n#\nA variety of modifications are applied to make YOLO prediction more accurate and faster, including:\n1. BatchNorm helps\n: Add\nbatch norm\non all the convolutional layers, leading to significant improvement over convergence.\n2. Image resolution matters\n: Fine-tuning the base model with\nhigh resolution\nimages improves the detection performance.\n3. Convolutional anchor box detection\n: Rather than predicts the bounding box position with fully-connected layers over the whole feature map, YOLOv2 uses\nconvolutional layers\nto predict locations of\nanchor boxes\n, like in faster R-CNN. The prediction of spatial locations and class probabilities are decoupled. Overall, the change leads to a slight decrease in mAP, but an increase in recall.\n4. K-mean clustering of box dimensions\n: Different from faster R-CNN that uses hand-picked sizes of anchor boxes, YOLOv2 runs k-mean clustering on the training data to find good priors on anchor box dimensions. The distance metric is designed to\nrely on IoU scores\n:\n$$\n\\text{dist}(x, c_i) = 1 - \\text{IoU}(x, c_i), i=1,\\dots,k\n$$\nwhere $x$ is a ground truth box candidate and $c_i$ is one of the centroids. The best number of centroids (anchor boxes) $k$ can be chosen by the\nelbow method\n.\nThe anchor boxes generated by clustering provide better average IoU conditioned on a fixed number of boxes.\n5. Direct location prediction\n: YOLOv2 formulates the bounding box prediction in a way that it would\nnot diverge\nfrom the center location too much. If the box location prediction can place the box in any part of the image, like in regional proposal network, the model training could become unstable.\nGiven the anchor box of size $(p_w, p_h)$ at the grid cell with its top left corner at $(c_x, c_y)$, the model predicts the offset and the scale, $(t_x, t_y, t_w, t_h)$ and the corresponding predicted bounding box $b$ has center $(b_x, b_y)$ and size $(b_w, b_h)$. The confidence score is the sigmoid ($\\sigma$) of another output $t_o$.\n$$\n\\begin{aligned}\nb_x &= \\sigma(t_x) + c_x\\\\\nb_y &= \\sigma(t_y) + c_y\\\\\nb_w &= p_w e^{t_w}\\\\\nb_h &= p_h e^{t_h}\\\\\n\\text{Pr}(\\text{object}) &\\cdot \\text{IoU}(b, \\text{object}) = \\sigma(t_o)\n\\end{aligned}\n$$\nYOLOv2 bounding box location prediction. (Image source:\noriginal paper\n)\n6. Add fine-grained features\n: YOLOv2 adds a passthrough layer to bring\nfine-grained features\nfrom an earlier layer to the last output layer. The mechanism of this passthrough layer is similar to\nidentity mappings in ResNet\nto extract higher-dimensional features from previous layers. This leads to 1% performance increase.\n7. Multi-scale training\n: In order to train the model to be robust to input images of different sizes, a\nnew size\nof input dimension is\nrandomly sampled\nevery 10 batches. Since conv layers of YOLOv2 downsample the input dimension by a factor of 32, the newly sampled size is a multiple of 32.\n8. Light-weighted base model\n: To make prediction even faster, YOLOv2 adopts a light-weighted base model, DarkNet-19, which has 19 conv layers and 5 max-pooling layers. The key point is to insert avg poolings and 1x1 conv filters between 3x3 conv layers.\nYOLO9000: Rich Dataset Training\n#\nBecause drawing bounding boxes on images for object detection is much more expensive than tagging images for classification, the paper proposed a way to combine small object detection dataset with large ImageNet so that the model can be exposed to a much larger number of object categories. The name of YOLO9000 comes from the top 9000 classes in ImageNet. During joint training, if an input image comes from the classification dataset, it only backpropagates the classification loss.\nThe detection dataset has much fewer and more general labels and, moreover, labels cross multiple datasets are often not mutually exclusive. For example, ImageNet has a label “Persian cat” while in COCO the same image would be labeled as “cat”. Without mutual exclusiveness, it does not make sense to apply softmax over all the classes.\nIn order to efficiently merge ImageNet labels (1000 classes, fine-grained) with COCO/PASCAL (< 100 classes, coarse-grained), YOLO9000 built a hierarchical tree structure with reference to\nWordNet\nso that general labels are closer to the root and the fine-grained class labels are leaves. In this way, “cat” is the parent node of “Persian cat”.\nThe WordTree hierarchy merges labels from COCO and ImageNet. Blue nodes are COCO labels and red nodes are ImageNet labels. (Image source:\noriginal paper\n)\nTo predict the probability of a class node, we can follow the path from the node to the root:\nPr(\"persian cat\" | contain a \"physical object\") \n= Pr(\"persian cat\" | \"cat\") \n Pr(\"cat\" | \"animal\") \n Pr(\"animal\" | \"physical object\") \n Pr(contain a \"physical object\") # confidence score.\nNote that\nPr(contain a \"physical object\")\nis the confidence score, predicted separately in the bounding box detection pipeline. The path of conditional probability prediction can stop at any step, depending on which labels are available.\nRetinaNet\n#\nThe\nRetinaNet\n(\nLin et al., 2018\n) is a one-stage dense object detector. Two crucial building blocks are\nfeaturized image pyramid\nand the use of\nfocal loss\n.\nFocal Loss\n#\nOne issue for object detection model training is an extreme imbalance between background that contains no object and foreground that holds objects of interests.\nFocal loss\nis designed to assign more weights on hard, easily misclassified examples (i.e. background with noisy texture or partial object) and to down-weight easy examples (i.e. obviously empty background).\nStarting with a normal cross entropy loss for binary classification,\n$$\n\\text{CE}(p, y) = -y\\log p - (1-y)\\log(1-p)\n$$\nwhere $y \\in \\{0, 1\\}$ is a ground truth binary label, indicating whether a bounding box contains a object, and $p \\in [0, 1]$ is the predicted probability of objectiveness (aka confidence score).\nFor notational convenience,\n$$\n\\text{let } p_t = \\begin{cases}\np & \\text{if } y = 1\\\\\n1-p & \\text{otherwise}\n\\end{cases},\n\\text{then } \\text{CE}(p, y)=\\text{CE}(p_t) = -\\log p_t\n$$\nEasily classified examples with large $p_t \\gg 0.5$, that is, when $p$ is very close to 0 (when y=0) or 1 (when y=1), can incur a loss with non-trivial magnitude. Focal loss explicitly adds a weighting factor $(1-p_t)^\\gamma, \\gamma \\geq 0$ to each term in cross entropy so that the weight is small when $p_t$ is large and therefore easy examples are down-weighted.\n$$\n\\text{FL}(p_t) = -(1-p_t)^\\gamma \\log p_t\n$$\nThe focal loss focuses less on easy examples with a factor of $(1-p\\_t)^\\gamma$. (Image source:\noriginal paper\n)\nFor a better control of the shape of the weighting function (see Fig. 10.), RetinaNet uses an $\\alpha$-balanced variant of the focal loss, where $\\alpha=0.25, \\gamma=2$ works the best.\n$$\n\\text{FL}(p_t) = -\\alpha (1-p_t)^\\gamma \\log p_t\n$$\nThe plot of focal loss weights $\\alpha (1-p\\_t)^\\gamma$ as a function of $p\\_t$, given different values of $\\alpha$ and $\\gamma$.\nFeaturized Image Pyramid\n#\nThe\nfeaturized image pyramid\n(\nLin et al., 2017\n) is the backbone network for RetinaNet. Following the same approach by\nimage pyramid\nin SSD, featurized image pyramids provide a basic vision component for object detection at different scales.\nThe key idea of feature pyramid network is demonstrated in The base structure contains a sequence of\npyramid levels\n, each corresponding to one network\nstage\n. One stage contains multiple convolutional layers of the same size and the stage sizes are scaled down by a factor of 2. Let’s denote the last layer of the $i$-th stage as $C_i$.\nThe illustration of the featurized image pyramid module. (Replot based on figure 3 in\nFPN paper\n)\nTwo pathways connect conv layers:\nBottom-up pathway\nis the normal feedforward computation.\nTop-down pathway\ngoes in the inverse direction, adding coarse but semantically stronger feature maps back into the previous pyramid levels of a larger size via lateral connections.\nFirst, the higher-level features are upsampled spatially coarser to be 2x larger. For image upscaling, the paper used nearest neighbor upsampling. While there are many\nimage upscaling algorithms\nsuch as using\ndeconv\n, adopting another image scaling method might or might not improve the performance of RetinaNet.\nThe larger feature map undergoes a 1x1 conv layer to reduce the channel dimension.\nFinally, these two feature maps are merged by element-wise addition.\nThe lateral connections only happen at the last layer in stages, denoted as $\\{C_i\\}$, and the process continues until the finest (largest) merged feature map is generated. The prediction is made out of every merged map after a 3x3 conv layer, $\\{P_i\\}$.\nAccording to ablation studies, the importance rank of components of the featurized image pyramid design is as follows:\n1x1 lateral connection\n> detect object across multiple layers > top-down enrichment > pyramid representation (compared to only check the finest layer).\nModel Architecture\n#\nThe featurized pyramid is constructed on top of the ResNet architecture. Recall that\nResNet\nhas 5 conv blocks (= network stages / pyramid levels). The last layer of the $i$-th pyramid level, $C_i$, has resolution $2^i$ lower than the raw input dimension.\nRetinaNet utilizes feature pyramid levels $P_3$ to $P_7$:\n$P_3$ to $P_5$ are computed from the corresponding ResNet residual stage from $C_3$ to $C_5$. They are connected by both top-down and bottom-up pathways.\n$P_6$ is obtained via a 3×3 stride-2 conv on top of $C_5$\n$P_7$ applies ReLU and a 3×3 stride-2 conv on $P_6$.\nAdding higher pyramid levels on ResNet improves the performance for detecting large objects.\nSame as in SSD, detection happens in all pyramid levels by making a prediction out of every merged feature map. Because predictions share the same classifier and the box regressor, they are all formed to have the same channel dimension d=256.\nThere are A=9 anchor boxes per level:\nThe base size corresponds to areas of $32^2$ to $512^2$ pixels on $P_3$ to $P_7$ respectively. There are three size ratios, $\\{2^0, 2^{1/3}, 2^{2/3}\\}$.\nFor each size, there are three aspect ratios {1/2, 1, 2}.\nAs usual, for each anchor box, the model outputs a class probability for each of $K$ classes in the classification subnet and regresses the offset from this anchor box to the nearest ground truth object in the box regression subnet. The classification subnet adopts the focal loss introduced above.\nThe RetinaNet model architecture uses a\nFPN\nbackbone on top of ResNet. (Image source: the\nFPN\npaper)\nYOLOv3\n#\nYOLOv3\nis created by applying a bunch of design tricks on YOLOv2. The changes are inspired by recent advances in the object detection world.\nHere are a list of changes:\n1. Logistic regression for confidence scores\n: YOLOv3 predicts an confidence score for each bounding box using\nlogistic regression\n, while YOLO and YOLOv2 uses sum of squared errors for classification terms (see the\nloss function\nabove). Linear regression of offset prediction leads to a decrease in mAP.\n2. No more softmax for class prediction\n: When predicting class confidence, YOLOv3 uses\nmultiple independent logistic classifier\nfor each class rather than one softmax layer. This is very helpful especially considering that one image might have multiple labels and not all the labels are guaranteed to be mutually exclusive.\n3. Darknet + ResNet as the base model\n: The new Darknet-53 still relies on successive 3x3 and 1x1 conv layers, just like the original dark net architecture, but has residual blocks added.\n4. Multi-scale prediction\n: Inspired by image pyramid, YOLOv3 adds several conv layers after the base feature extractor model and makes prediction at three different scales among these conv layers. In this way, it has to deal with many more bounding box candidates of various sizes overall.\n5. Skip-layer concatenation\n: YOLOv3 also adds cross-layer connections between two prediction layers (except for the output layer) and earlier finer-grained feature maps. The model first up-samples the coarse feature maps and then merges it with the previous features by concatenation. The combination with finer-grained information makes it better at detecting small objects.\nInterestingly, focal loss does not help YOLOv3, potentially it might be due to the usage of $\\lambda_\\text{noobj}$ and $\\lambda_\\text{coord}$ — they increase the loss from bounding box location predictions and decrease the loss from confidence predictions for background boxes.\nOverall YOLOv3 performs better and faster than SSD, and worse than RetinaNet but 3.8x faster.\nThe comparison of various fast object detection models on speed and mAP performance. (Image source:\nfocal loss\npaper with additional labels from the\nYOLOv3\npaper.)\nCited as:\n@article{weng2018detection4,\n title = \"Object Detection Part 4: Fast Detection Models\",\n author = \"Weng, Lilian\",\n journal = \"lilianweng.github.io\",\n year = \"2018\",\n url = \"https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/\"\n}\nReference\n#\n[1] Joseph Redmon, et al.\n“You only look once: Unified, real-time object detection.”\nCVPR 2016.\n[2] Joseph Redmon and Ali Farhadi.\n“YOLO9000: Better, Faster, Stronger.”\nCVPR 2017.\n[3] Joseph Redmon, Ali Farhadi.\n“YOLOv3: An incremental improvement.”\n.\n[4] Wei Liu et al.\n“SSD: Single Shot MultiBox Detector.”\nECCV 2016.\n[5] Tsung-Yi Lin, et al.\n“Feature Pyramid Networks for Object Detection.”\nCVPR 2017.\n[6] Tsung-Yi Lin, et al.\n“Focal Loss for Dense Object Detection.”\nIEEE transactions on pattern analysis and machine intelligence, 2018.\n[7]\n“What’s new in YOLO v3?”\nby Ayoosh Kathuria on “Towards Data Science”, Apr 23, 2018.", "repair_version": 2, "repaired_at": "2026-10-14T11:45:43Z", "title": "Object Detection Part 4: Fast Detection Models | Lil'Log", "audio": [], "code_blocks": [ { "file": "attachments/466144a71db1/code/body-000.txt", "chars": 195, "language": "text" }, { "file": "attachments/466144a71db1/code/body-001.txt", "chars": 264, "language": "text" } ], "papers": [ { "status": "ok", "url": "https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Redmon_You_Only_Look_CVPR_2016_paper.pdf", "bytes": 1270136, "reused": true, "local_path": "papers/repair/326369cb8bd278bd904e.pdf" }, { "status": "http_404", "url": "https://www.cs.unc.edu/~wliu/papers/GoogLeNet.pdf", "retried_at": "2026-10-20T17:55:04Z" }, { "status": "ok", "url": "https://arxiv.org/pdf/1512.02325", "bytes": 2494225, "reused": true, "local_path": "papers/repair/b5f5e9ca4f7f92190f2f.pdf" }, { "status": "ok", "url": "https://arxiv.org/pdf/1409.1556", "bytes": 200010, "reused": true, "local_path": "papers/repair/01560c40deef0c339e01.pdf" }, { "status": "ok", "url": "https://github.com/rbgirshick/py-faster-rcnn/files/764206/SmoothL1Loss.1.pdf", "bytes": 132365, "reused": true, "local_path": "papers/repair/75b4690976ed5ab1f64d.pdf" }, { "status": "ok", "url": "https://arxiv.org/pdf/1612.08242", "bytes": 5256083, "reused": true, "local_path": "papers/repair/1288fc7e9385e0345e39.pdf" }, { "status": "ok", "url": "https://arxiv.org/pdf/1708.02002", "bytes": 1297358, "reused": false, "content_type": "application/pdf", "local_path": "papers/repair/c9512e3ffac1aabce76e.pdf" }, { "status": "ok", "url": "https://arxiv.org/pdf/1612.03144", "bytes": 789142, "reused": false, "content_type": "application/pdf", "local_path": "papers/repair/0cd1e11096f8a72ff39b.pdf" }, { "status": "ok", "url": "https://pjreddie.com/media/files/papers/YOLOv3.pdf", "bytes": 2241388, "reused": false, "content_type": "application/pdf", "local_path": "papers/repair/5bc4175dabe4accf71a9.pdf" }, { "status": "ok", "url": "http://openaccess.thecvf.com/content_cvpr_2017/papers/Redmon_YOLO9000_Better_Faster_CVPR_2017_paper.pdf", "bytes": 5392639, "reused": false, "content_type": "application/pdf", "local_path": "papers/repair/edb99b1cb9792a271703.pdf" } ], "actual_video_files": [], "body_chars": 25701, "expected_body_images": 13, "discovery_status": "ok", "delivery_status": "partial", "missing_reason": "http_404", "raw_html": "attachments/466144a71db1/pages/source.html", "local_html": "attachments/466144a71db1/pages/article.html", "body_base_path": "attachments/466144a71db1/pages/", "body_markdown_path": "attachments/466144a71db1/pages/article.md", "links": [ { "href": "https://lilianweng.github.io/posts/2017-12-31-object-recognition-part-3/", "label": "Part 3", "type": "a" }, { "href": "https://lilianweng.github.io/posts/2017-10-29-object-recognition-part-1/", "label": "Part 1", "type": "a" }, { "href": "https://lilianweng.github.io/posts/2017-12-15-object-recognition-part-2/", "label": "Part 2", "type": "a" }, { "href": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/", "label": "Part 4", "type": "a" }, { "href": "https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Redmon_You_Only_Look_CVPR_2016_paper.pdf", "label": "Redmon et al., 2016", "type": "a" }, { "href": "https://www.cs.unc.edu/~wliu/papers/GoogLeNet.pdf", "label": "GoogLeNet", "type": "a" }, { "href": "https://arxiv.org/abs/1512.02325", "label": "Liu et al, 2016", "type": "a" }, { "href": "https://arxiv.org/abs/1409.1556", "label": "VGG-16", "type": "a" }, { "href": "https://github.com/rbgirshick/py-faster-rcnn/files/764206/SmoothL1Loss.1.pdf", "label": "smooth L1 loss", "type": "a" }, { "href": "https://www.tensorflow.org/api_docs/python/tf/nn/softmax_cross_entropy_with_logits", "label": "softmax_cross_entropy_with_logits", "type": "a" }, { "href": "https://arxiv.org/abs/1612.08242", "label": "Redmon & Farhadi, 2017", "type": "a" }, { "href": "https://en.wikipedia.org/wiki/Elbow_method_(clustering)", "label": "elbow method", "type": "a" }, { "href": "https://wordnet.princeton.edu/", "label": "WordNet", "type": "a" }, { "href": "https://arxiv.org/abs/1708.02002", "label": "Lin et al., 2018", "type": "a" }, { "href": "https://arxiv.org/abs/1612.03144", "label": "Lin et al., 2017", "type": "a" }, { "href": "https://en.wikipedia.org/wiki/Image_scaling", "label": "image upscaling algorithms", "type": "a" }, { "href": "https://www.tensorflow.org/api_docs/python/tf/layers/conv2d_transpose", "label": "deconv", "type": "a" }, { "href": "https://lilianweng.github.io/posts/2018-12-27-object-recognition-part-4/TBA", "label": "ResNet", "type": "a" }, { "href": "https://pjreddie.com/media/files/papers/YOLOv3.pdf", "label": "YOLOv3", "type": "a" }, { "href": "http://openaccess.thecvf.com/content_cvpr_2017/papers/Redmon_YOLO9000_Better_Faster_CVPR_2017_paper.pdf", "label": "“YOLO9000: Better, Faster, Stronger.”", "type": "a" }, { "href": "https://towardsdatascience.com/yolo-v3-object-detection-53fb7d3bfe6b", "label": "“What’s new in YOLO v3?”", "type": "a" } ], "crawl": { "success": true, "status_code": 200, "error": "" } }