Skip to content

Reduce memory usage peak when using thumbnail animated WebP decoding and encoding #73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 16, 2023
Merged

Reduce memory usage peak when using thumbnail animated WebP decoding and encoding #73

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0b25f9a
Support using CoreGraphics to produce scaled down CGImage directly
dreampiggy Mar 8, 2023
c779312
Use kvImageNoAllocate to avoid extra allocate for vImageBuffer when W…
dreampiggy Mar 8, 2023
9dae8d3
Refactory thumbnail decoding, use a transformed CGContext instead of …
dreampiggy Mar 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 50 additions & 111 deletions SDWebImageWebPCoder/Classes/SDImageWebPCoder.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,26 @@
#endif
#endif

/// Used for animated WebP, which need a canvas for decoding (rendering), possible apply a scale transform for thumbnail decoding (avoiding post-rescale using vImage)
/// See more in #73
static inline CGContextRef _Nullable CreateWebPCanvas(BOOL hasAlpha, CGSize canvasSize, CGSize thumbnailSize, BOOL preserveAspectRatio) {
CGBitmapInfo bitmapInfo = kCGBitmapByteOrder32Host;
bitmapInfo |= hasAlpha ? kCGImageAlphaPremultipliedFirst : kCGImageAlphaNoneSkipFirst;
// Check whether we need to use thumbnail
CGSize scaledSize = [SDImageCoderHelper scaledSizeWithImageSize:CGSizeMake(canvasSize.width, canvasSize.height) scaleSize:thumbnailSize preserveAspectRatio:preserveAspectRatio shouldScaleUp:NO];
CGContextRef canvas = CGBitmapContextCreate(NULL, scaledSize.width, scaledSize.height, 8, 0, [SDImageCoderHelper colorSpaceGetDeviceRGB], bitmapInfo);
if (!canvas) {
return nil;
}
// Check whether we need to use thumbnail
if (!CGSizeEqualToSize(canvasSize, scaledSize)) {
CGFloat sx = scaledSize.width / canvasSize.width;
CGFloat sy = scaledSize.height / canvasSize.height;
CGContextScaleCTM(canvas, sx, sy);
}
return canvas;
}

@interface SDWebPCoderFrame : NSObject

@property (nonatomic, assign) NSUInteger index; // Frame index (zero based)
Expand Down Expand Up @@ -218,9 +238,7 @@ - (UIImage *)decodedImageWithData:(NSData *)data options:(nullable SDImageCoderO
}

BOOL hasAlpha = flags & ALPHA_FLAG;
CGBitmapInfo bitmapInfo = kCGBitmapByteOrder32Host;
bitmapInfo |= hasAlpha ? kCGImageAlphaPremultipliedFirst : kCGImageAlphaNoneSkipFirst;
CGContextRef canvas = CGBitmapContextCreate(NULL, canvasWidth, canvasHeight, 8, 0, [SDImageCoderHelper colorSpaceGetDeviceRGB], bitmapInfo);
CGContextRef canvas = CreateWebPCanvas(hasAlpha, CGSizeMake(canvasWidth, canvasHeight), thumbnailSize, preserveAspectRatio);
if (!canvas) {
WebPDemuxDelete(demuxer);
CGColorSpaceRelease(colorSpace);
Expand All @@ -232,7 +250,7 @@ - (UIImage *)decodedImageWithData:(NSData *)data options:(nullable SDImageCoderO

do {
@autoreleasepool {
CGImageRef imageRef = [self sd_drawnWebpImageWithCanvas:canvas iterator:iter colorSpace:colorSpace scaledSize:scaledSize];
CGImageRef imageRef = [self sd_drawnWebpImageWithCanvas:canvas demuxer:demuxer iterator:iter colorSpace:colorSpace];
if (!imageRef) {
continue;
}
Expand Down Expand Up @@ -381,7 +399,7 @@ - (UIImage *)incrementalDecodedImageWithOptions:(SDImageCoderOptions *)options {
return nil;
}

CGContextRef canvas = CGBitmapContextCreate(NULL, width, height, 8, 0, [SDImageCoderHelper colorSpaceGetDeviceRGB], bitmapInfo);
CGContextRef canvas = CreateWebPCanvas(YES, CGSizeMake(width, height), _thumbnailSize, _preserveAspectRatio);
if (!canvas) {
CGImageRelease(imageRef);
return nil;
Expand All @@ -403,13 +421,6 @@ - (UIImage *)incrementalDecodedImageWithOptions:(SDImageCoderOptions *)options {
scale = 1;
}
}
CGSize scaledSize = [SDImageCoderHelper scaledSizeWithImageSize:CGSizeMake(width, height) scaleSize:_thumbnailSize preserveAspectRatio:_preserveAspectRatio shouldScaleUp:NO];
// Check whether we need to use thumbnail
if (!CGSizeEqualToSize(CGSizeMake(width, height), scaledSize)) {
CGImageRef scaledImageRef = [SDImageCoderHelper CGImageCreateScaled:newImageRef size:scaledSize];
CGImageRelease(newImageRef);
newImageRef = scaledImageRef;
}

#if SD_UIKIT || SD_WATCH
image = [[UIImage alloc] initWithCGImage:newImageRef scale:scale orientation:UIImageOrientationUp];
Expand All @@ -425,8 +436,8 @@ - (UIImage *)incrementalDecodedImageWithOptions:(SDImageCoderOptions *)options {
return image;
}

- (void)sd_blendWebpImageWithCanvas:(CGContextRef)canvas iterator:(WebPIterator)iter colorSpace:(nonnull CGColorSpaceRef)colorSpaceRef {
size_t canvasHeight = CGBitmapContextGetHeight(canvas);
- (void)sd_blendWebpImageWithCanvas:(CGContextRef)canvas demuxer:(nonnull WebPDemuxer *)demuxer iterator:(WebPIterator)iter colorSpace:(nonnull CGColorSpaceRef)colorSpaceRef {
int canvasHeight = WebPDemuxGetI(demuxer, WEBP_FF_CANVAS_HEIGHT);
CGFloat tmpX = iter.x_offset;
CGFloat tmpY = canvasHeight - iter.height - iter.y_offset;
CGRect imageRect = CGRectMake(tmpX, tmpY, iter.width, iter.height);
Expand All @@ -448,14 +459,13 @@ - (void)sd_blendWebpImageWithCanvas:(CGContextRef)canvas iterator:(WebPIterator)
}
}

- (nullable CGImageRef)sd_drawnWebpImageWithCanvas:(CGContextRef)canvas iterator:(WebPIterator)iter colorSpace:(nonnull CGColorSpaceRef)colorSpaceRef scaledSize:(CGSize)scaledSize CF_RETURNS_RETAINED {
- (nullable CGImageRef)sd_drawnWebpImageWithCanvas:(CGContextRef)canvas demuxer:(nonnull WebPDemuxer *)demuxer iterator:(WebPIterator)iter colorSpace:(nonnull CGColorSpaceRef)colorSpaceRef CF_RETURNS_RETAINED {
CGImageRef imageRef = [self sd_createWebpImageWithData:iter.fragment colorSpace:colorSpaceRef scaledSize:CGSizeZero];
if (!imageRef) {
return nil;
}

size_t canvasWidth = CGBitmapContextGetWidth(canvas);
size_t canvasHeight = CGBitmapContextGetHeight(canvas);
int canvasHeight = WebPDemuxGetI(demuxer, WEBP_FF_CANVAS_HEIGHT);
CGFloat tmpX = iter.x_offset;
CGFloat tmpY = canvasHeight - iter.height - iter.y_offset;
CGRect imageRect = CGRectMake(tmpX, tmpY, iter.width, iter.height);
Expand All @@ -466,26 +476,15 @@ - (nullable CGImageRef)sd_drawnWebpImageWithCanvas:(CGContextRef)canvas iterator
if (!shouldBlend) {
CGContextClearRect(canvas, imageRect);
}

CGContextDrawImage(canvas, imageRect, imageRef);
CGImageRef newImageRef = CGBitmapContextCreateImage(canvas);

CGImageRelease(imageRef);

if (iter.dispose_method == WEBP_MUX_DISPOSE_BACKGROUND) {
CGContextClearRect(canvas, imageRect);
}

// Check whether we need to use thumbnail
if (!CGSizeEqualToSize(CGSizeMake(canvasWidth, canvasHeight), scaledSize)) {
// Important: For Animated WebP thumbnail generation, we can not just use a scaled small canvas and draw each thumbnail frame
// This works **On Theory**. However, image scale down loss details. Animated WebP use the partial pixels with blend mode / dispose method with offset, to cover previous canvas status
// Because of this reason, even each frame contains small zigzag, the final animation contains visible glitch, this is not we want.
// So, always create the full pixels canvas (even though this consume more RAM), after drawn on the canvas, re-scale again with the final size
CGImageRef scaledImageRef = [SDImageCoderHelper CGImageCreateScaled:newImageRef size:scaledSize];
CGImageRelease(newImageRef);
newImageRef = scaledImageRef;
}

return newImageRef;
}

Expand Down Expand Up @@ -736,76 +735,22 @@ - (nullable NSData *)sd_encodedWebpDataWithImage:(nullable CGImageRef)imageRef
if (!dataProvider) {
return nil;
}
// Check colorSpace is RGB/RGBA
CGColorSpaceRef colorSpace = CGImageGetColorSpace(imageRef);
BOOL isRGB = CGColorSpaceGetModel(colorSpace) == kCGColorSpaceModelRGB;

CFDataRef dataRef;
uint8_t *rgba = NULL; // RGBA Buffer managed by CFData, don't call `free` on it, instead call `CFRelease` on `dataRef`
// We could not assume that input CGImage's color mode is always RGB888/RGBA8888. Convert all other cases to target color mode using vImage
BOOL isRGB888 = isRGB && byteOrderNormal && alphaInfo == kCGImageAlphaNone && components == 3;
BOOL isRGBA8888 = isRGB && byteOrderNormal && alphaInfo == kCGImageAlphaLast && components == 4;
if (isRGB888 || isRGBA8888) {
// If the input CGImage is already RGB888/RGBA8888
dataRef = CGDataProviderCopyData(dataProvider);
if (!dataRef) {
return nil;
}
rgba = (uint8_t *)CFDataGetBytePtr(dataRef);
} else {
// Convert all other cases to target color mode using vImage
vImageConverterRef convertor = NULL;
vImage_Error error = kvImageNoError;

vImage_CGImageFormat srcFormat = {
.bitsPerComponent = (uint32_t)bitsPerComponent,
.bitsPerPixel = (uint32_t)bitsPerPixel,
.colorSpace = colorSpace,
.bitmapInfo = bitmapInfo,
.renderingIntent = CGImageGetRenderingIntent(imageRef)
};
vImage_CGImageFormat destFormat = {
.bitsPerComponent = 8,
.bitsPerPixel = hasAlpha ? 32 : 24,
.colorSpace = [SDImageCoderHelper colorSpaceGetDeviceRGB],
.bitmapInfo = hasAlpha ? kCGImageAlphaLast | kCGBitmapByteOrderDefault : kCGImageAlphaNone | kCGBitmapByteOrderDefault // RGB888/RGBA8888 (Non-premultiplied to works for libwebp)
};

convertor = vImageConverter_CreateWithCGImageFormat(&srcFormat, &destFormat, NULL, kvImageNoFlags, &error);
if (error != kvImageNoError) {
return nil;
}

vImage_Buffer src;
error = vImageBuffer_InitWithCGImage(&src, &srcFormat, nil, imageRef, kvImageNoFlags);
if (error != kvImageNoError) {
vImageConverter_Release(convertor);
return nil;
}

vImage_Buffer dest;
error = vImageBuffer_Init(&dest, height, width, destFormat.bitsPerPixel, kvImageNoFlags);
if (error != kvImageNoError) {
vImageConverter_Release(convertor);
free(src.data);
return nil;
}

// Convert input color mode to RGB888/RGBA8888
error = vImageConvert_AnyToAny(convertor, &src, &dest, NULL, kvImageNoFlags);

// Free the buffer
free(src.data);
vImageConverter_Release(convertor);
if (error != kvImageNoError) {
free(dest.data);
return nil;
}

rgba = dest.data; // Converted buffer
bytesPerRow = dest.rowBytes; // Converted bytePerRow
dataRef = CFDataCreateWithBytesNoCopy(kCFAllocatorDefault, rgba, bytesPerRow * height, kCFAllocatorDefault);
vImage_CGImageFormat destFormat = {
.bitsPerComponent = 8,
.bitsPerPixel = hasAlpha ? 32 : 24,
.colorSpace = [SDImageCoderHelper colorSpaceGetDeviceRGB],
.bitmapInfo = hasAlpha ? kCGImageAlphaLast | kCGBitmapByteOrderDefault : kCGImageAlphaNone | kCGBitmapByteOrderDefault // RGB888/RGBA8888 (Non-premultiplied to works for libwebp)
};
vImage_Buffer dest;
vImage_Error error = vImageBuffer_InitWithCGImage(&dest, &destFormat, NULL, imageRef, kvImageNoFlags);
if (error != kvImageNoError) {
return nil;
}
rgba = dest.data;
bytesPerRow = dest.rowBytes;

float qualityFactor = quality * 100; // WebP quality is 0-100
// Encode RGB888/RGBA8888 buffer to WebP data
Expand All @@ -817,7 +762,8 @@ - (nullable NSData *)sd_encodedWebpDataWithImage:(nullable CGImageRef)imageRef
if (!WebPConfigPreset(&config, WEBP_PRESET_DEFAULT, qualityFactor) ||
!WebPPictureInit(&picture)) {
// shouldn't happen, except if system installation is broken
CFRelease(dataRef);
free(dest.data);
// CFRelease(dataRef);
return nil;
}

Expand All @@ -837,7 +783,7 @@ - (nullable NSData *)sd_encodedWebpDataWithImage:(nullable CGImageRef)imageRef
}
if (!result) {
WebPMemoryWriterClear(&writer);
CFRelease(dataRef);
free(dest.data);
return nil;
}

Expand All @@ -848,14 +794,14 @@ - (nullable NSData *)sd_encodedWebpDataWithImage:(nullable CGImageRef)imageRef
if (!result) {
WebPMemoryWriterClear(&writer);
WebPPictureFree(&picture);
CFRelease(dataRef);
free(dest.data);
return nil;
}
}

result = WebPEncode(&config, &picture);
WebPPictureFree(&picture);
CFRelease(dataRef); // Free bitmap buffer
free(dest.data);

if (result) {
// success
Expand Down Expand Up @@ -1137,16 +1083,13 @@ - (UIImage *)safeStaticImageFrame {
if (_hasAnimation) {
// If have animation, we still need to allocate a CGContext, because the poster frame may be smaller than canvas
if (!_canvas) {
CGBitmapInfo bitmapInfo = kCGBitmapByteOrder32Host;
bitmapInfo |= _hasAlpha ? kCGImageAlphaPremultipliedFirst : kCGImageAlphaNoneSkipFirst;
CGContextRef canvas = CGBitmapContextCreate(NULL, _canvasWidth, _canvasHeight, 8, 0, [SDImageCoderHelper colorSpaceGetDeviceRGB], bitmapInfo);
CGContextRef canvas = CreateWebPCanvas(_hasAlpha, CGSizeMake(_canvasWidth, _canvasHeight), _thumbnailSize, _preserveAspectRatio);
if (!canvas) {
return nil;
}
_canvas = canvas;
}
CGSize scaledSize = [SDImageCoderHelper scaledSizeWithImageSize:CGSizeMake(_canvasWidth, _canvasHeight) scaleSize:_thumbnailSize preserveAspectRatio:_preserveAspectRatio shouldScaleUp:NO];
imageRef = [self sd_drawnWebpImageWithCanvas:_canvas iterator:iter colorSpace:_colorSpace scaledSize:scaledSize];
imageRef = [self sd_drawnWebpImageWithCanvas:_canvas demuxer:_demux iterator:iter colorSpace:_colorSpace];
} else {
CGSize scaledSize = [SDImageCoderHelper scaledSizeWithImageSize:CGSizeMake(iter.width, iter.height) scaleSize:_thumbnailSize preserveAspectRatio:_preserveAspectRatio shouldScaleUp:NO];
imageRef = [self sd_createWebpImageWithData:iter.fragment colorSpace:_colorSpace scaledSize:scaledSize];
Expand All @@ -1166,9 +1109,7 @@ - (UIImage *)safeStaticImageFrame {

- (UIImage *)safeAnimatedImageFrameAtIndex:(NSUInteger)index {
if (!_canvas) {
CGBitmapInfo bitmapInfo = kCGBitmapByteOrder32Host;
bitmapInfo |= _hasAlpha ? kCGImageAlphaPremultipliedFirst : kCGImageAlphaNoneSkipFirst;
CGContextRef canvas = CGBitmapContextCreate(NULL, _canvasWidth, _canvasHeight, 8, 0, [SDImageCoderHelper colorSpaceGetDeviceRGB], bitmapInfo);
CGContextRef canvas = CreateWebPCanvas(_hasAlpha, CGSizeMake(_canvasWidth, _canvasHeight), _thumbnailSize, _preserveAspectRatio);
if (!canvas) {
return nil;
}
Expand Down Expand Up @@ -1212,7 +1153,7 @@ - (UIImage *)safeAnimatedImageFrameAtIndex:(NSUInteger)index {
if (endIndex > startIndex) {
do {
@autoreleasepool {
[self sd_blendWebpImageWithCanvas:_canvas iterator:iter colorSpace:_colorSpace];
[self sd_blendWebpImageWithCanvas:_canvas demuxer:_demux iterator:iter colorSpace:_colorSpace];
}
} while ((size_t)iter.frame_num < endIndex && WebPDemuxNextFrame(&iter));
}
Expand All @@ -1225,9 +1166,7 @@ - (UIImage *)safeAnimatedImageFrameAtIndex:(NSUInteger)index {
_currentBlendIndex = index;

// Now the canvas is ready, which respects of dispose method behavior. Just do normal decoding and produce image.
// Check whether we need to use thumbnail
CGSize scaledSize = [SDImageCoderHelper scaledSizeWithImageSize:CGSizeMake(_canvasWidth, _canvasHeight) scaleSize:_thumbnailSize preserveAspectRatio:_preserveAspectRatio shouldScaleUp:NO];
CGImageRef imageRef = [self sd_drawnWebpImageWithCanvas:_canvas iterator:iter colorSpace:_colorSpace scaledSize:scaledSize];
CGImageRef imageRef = [self sd_drawnWebpImageWithCanvas:_canvas demuxer:_demux iterator:iter colorSpace:_colorSpace];
if (!imageRef) {
return nil;
}
Expand Down